From b4d0280702689753276a4058433644c2dfa1ff43 Mon Sep 17 00:00:00 2001 From: Anton Kosyakov Date: Fri, 10 Jul 2020 13:22:32 +0000 Subject: [PATCH] [monaco] stream content to avoid blocking the backend and network by large files Signed-off-by: Anton Kosyakov --- packages/core/src/common/resource.ts | 39 +++++++++++-- packages/core/src/common/stream.ts | 8 +++ .../src/browser/editor-preview-manager.ts | 11 +++- .../filesystem/src/browser/file-resource.ts | 57 ++++++++++++++++++- .../monaco/src/browser/monaco-editor-model.ts | 57 ++++++++++++------- packages/monaco/src/browser/monaco-loader.ts | 6 +- packages/monaco/src/typings/monaco/index.d.ts | 31 ++++++++++ 7 files changed, 177 insertions(+), 32 deletions(-) diff --git a/packages/core/src/common/resource.ts b/packages/core/src/common/resource.ts index 6530f4c7b8e2e..ffec243ec0ead 100644 --- a/packages/core/src/common/resource.ts +++ b/packages/core/src/common/resource.ts @@ -23,6 +23,7 @@ import { Disposable } from './disposable'; import { MaybePromise } from './types'; import { CancellationToken } from './cancellation'; import { ApplicationError } from './application-error'; +import { ReadableStream, Readable } from './stream'; export interface ResourceVersion { } @@ -62,6 +63,15 @@ export interface Resource extends Disposable { * @throws `ResourceError.NotFound` if a resource not found */ readContents(options?: ResourceReadOptions): Promise; + /** + * Stream latest content of this resource. + * + * If a resource supports versioning it updates version to latest. + * If a resource supports encoding it updates encoding to latest. + * + * @throws `ResourceError.NotFound` if a resource not found + */ + readStream?(options?: ResourceReadOptions): Promise>; /** * Rewrites the complete content for this resource. * If a resource does not exist it will be created. @@ -74,6 +84,18 @@ export interface Resource extends Disposable { * @throws `ResourceError.OutOfSync` if latest resource version is out of sync with the given */ saveContents?(content: string, options?: ResourceSaveOptions): Promise; + /** + * Rewrites the complete content for this resource. + * If a resource does not exist it will be created. + * + * If a resource supports versioning clients can pass some version + * to check against it, if it is not provided latest version is used. + * + * It updates version and encoding to latest. + * + * @throws `ResourceError.OutOfSync` if latest resource version is out of sync with the given + */ + saveStream?(content: Readable, options?: ResourceSaveOptions): Promise; /** * Applies incremental content changes to this resource. * @@ -90,7 +112,8 @@ export interface Resource extends Disposable { } export namespace Resource { export interface SaveContext { - content: string + contentLength: number + content: string | Readable changes?: TextDocumentContentChangeEvent[] options?: ResourceSaveOptions } @@ -104,10 +127,15 @@ export namespace Resource { if (token && token.isCancellationRequested) { return; } - await resource.saveContents(context.content, context.options); + if (typeof context.content !== 'string' && resource.saveStream) { + await resource.saveStream(context.content, context.options); + } else { + const content = typeof context.content === 'string' ? context.content : Readable.toString(context.content); + await resource.saveContents(content, context.options); + } } export async function trySaveContentChanges(resource: Resource, context: SaveContext): Promise { - if (!context.changes || !resource.saveContentChanges || shouldSaveContent(context)) { + if (!context.changes || !resource.saveContentChanges || shouldSaveContent(resource, context)) { return false; } try { @@ -120,12 +148,11 @@ export namespace Resource { return false; } } - export function shouldSaveContent({ content, changes }: SaveContext): boolean { - if (!changes) { + export function shouldSaveContent(resource: Resource, { contentLength, changes }: SaveContext): boolean { + if (!changes || (resource.saveStream && contentLength > 32 * 1024 * 1024)) { return true; } let contentChangesLength = 0; - const contentLength = content.length; for (const change of changes) { contentChangesLength += JSON.stringify(change).length; if (contentChangesLength > contentLength) { diff --git a/packages/core/src/common/stream.ts b/packages/core/src/common/stream.ts index 615065d6f8054..3f5131d2e4293 100644 --- a/packages/core/src/common/stream.ts +++ b/packages/core/src/common/stream.ts @@ -103,6 +103,14 @@ export namespace Readable { } }; } + export function toString(readable: Readable): string { + let result = ''; + let chunk: string | null; + while ((chunk = readable.read()) != null) { + result += chunk; + } + return result; + } } /** diff --git a/packages/editor-preview/src/browser/editor-preview-manager.ts b/packages/editor-preview/src/browser/editor-preview-manager.ts index 2079e1ac257b1..2515fa59e7b67 100644 --- a/packages/editor-preview/src/browser/editor-preview-manager.ts +++ b/packages/editor-preview/src/browser/editor-preview-manager.ts @@ -140,8 +140,15 @@ export class EditorPreviewManager extends WidgetOpenHandler { - return this.currentEditorPreview = super.open(uri, options) as Promise; + protected openNewPreview(uri: URI, options: PreviewEditorOpenerOptions): Promise { + const result = super.open(uri, options); + this.currentEditorPreview = result.then(widget => { + if (widget instanceof EditorPreviewWidget) { + return widget; + } + return undefined; + }, () => undefined); + return result; } protected createWidgetOptions(uri: URI, options?: WidgetOpenerOptions): EditorPreviewWidgetOptions { diff --git a/packages/filesystem/src/browser/file-resource.ts b/packages/filesystem/src/browser/file-resource.ts index 274267dfaea88..ccb98f764e747 100644 --- a/packages/filesystem/src/browser/file-resource.ts +++ b/packages/filesystem/src/browser/file-resource.ts @@ -18,6 +18,7 @@ import { injectable, inject } from 'inversify'; import { Resource, ResourceVersion, ResourceResolver, ResourceError, ResourceSaveOptions } from '@theia/core/lib/common/resource'; import { DisposableCollection } from '@theia/core/lib/common/disposable'; import { Emitter, Event } from '@theia/core/lib/common/event'; +import { Readable, ReadableStream } from '@theia/core/lib/common/stream'; import URI from '@theia/core/lib/common/uri'; import { FileOperation, FileOperationError, FileOperationResult, ETAG_DISABLED, FileSystemProviderCapabilities, FileReadStreamOptions, BinarySize } from '../common/files'; import { FileService, TextFileOperationError, TextFileOperationResult } from './file-service'; @@ -135,7 +136,59 @@ export class FileResource implements Resource { } } - async saveContents(content: string, options?: ResourceSaveOptions): Promise { + async readStream(options?: { encoding?: string }): Promise> { + try { + const encoding = options?.encoding || this.version?.encoding; + const stat = await this.fileService.readStream(this.uri, { + encoding, + etag: ETAG_DISABLED, + acceptTextOnly: this.acceptTextOnly, + limits: this.limits + }); + this._version = { + encoding: stat.encoding, + etag: stat.etag, + mtime: stat.mtime + }; + return stat.value; + } catch (e) { + if (e instanceof TextFileOperationError && e.textFileOperationResult === TextFileOperationResult.FILE_IS_BINARY) { + if (await this.shouldOpenAsText('The file is either binary or uses an unsupported text encoding.')) { + this.acceptTextOnly = false; + return this.readStream(options); + } + } else if (e instanceof FileOperationError && e.fileOperationResult === FileOperationResult.FILE_TOO_LARGE) { + const stat = await this.fileService.resolve(this.uri, { resolveMetadata: true }); + const maxFileSize = GENERAL_MAX_FILE_SIZE_MB * 1024 * 1024; + if (this.limits?.size !== maxFileSize && await this.shouldOpenAsText(`The file is too large (${BinarySize.formatSize(stat.size)}).`)) { + this.limits = { + size: maxFileSize + }; + return this.readStream(options); + } + } else if (e instanceof FileOperationError && e.fileOperationResult === FileOperationResult.FILE_NOT_FOUND) { + this._version = undefined; + const { message, stack } = e; + throw ResourceError.NotFound({ + message, stack, + data: { + uri: this.uri + } + }); + } + throw e; + } + } + + saveContents(content: string, options?: ResourceSaveOptions): Promise { + return this.doWrite(content, options); + } + + saveStream(content: Readable, options?: ResourceSaveOptions): Promise { + return this.doWrite(content, options); + } + + protected async doWrite(content: string | Readable, options?: ResourceSaveOptions): Promise { const version = options?.version || this._version; const current = FileResourceVersion.is(version) ? version : undefined; const etag = current?.etag; @@ -154,7 +207,7 @@ export class FileResource implements Resource { } catch (e) { if (e instanceof FileOperationError && e.fileOperationResult === FileOperationResult.FILE_MODIFIED_SINCE) { if (etag !== ETAG_DISABLED && await this.shouldOverwrite()) { - return this.saveContents(content, { ...options, version: { stat: { ...current, etag: ETAG_DISABLED } } }); + return this.doWrite(content, { ...options, version: { stat: { ...current, etag: ETAG_DISABLED } } }); } const { message, stack } = e; throw ResourceError.OutOfSync({ message, stack, data: { uri: this.uri } }); diff --git a/packages/monaco/src/browser/monaco-editor-model.ts b/packages/monaco/src/browser/monaco-editor-model.ts index 22e7fac12d522..6e45ecf41bd87 100644 --- a/packages/monaco/src/browser/monaco-editor-model.ts +++ b/packages/monaco/src/browser/monaco-editor-model.ts @@ -92,8 +92,7 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument { this.toDispose.push(Disposable.create(() => this.cancelSave())); this.toDispose.push(Disposable.create(() => this.cancelSync())); this.resolveModel = this.readContents().then( - content => this.initialize(content || ''), - e => console.error(`Failed to initialize for '${this.resource.uri.toString()}':`, e) + content => this.initialize(content || '') ); } @@ -143,9 +142,21 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument { * Only this method can create an instance of `monaco.editor.IModel`, * there should not be other calls to `monaco.editor.createModel`. */ - protected initialize(content: string): void { + protected initialize(value: string | monaco.editor.ITextBufferFactory): void { if (!this.toDispose.disposed) { - this.model = monaco.editor.createModel(content, undefined, monaco.Uri.parse(this.resource.uri.toString())); + const uri = monaco.Uri.parse(this.resource.uri.toString()); + let firstLine; + if (typeof value === 'string') { + firstLine = value; + const firstLF = value.indexOf('\n'); + if (firstLF !== -1) { + firstLine = value.substring(0, firstLF); + } + } else { + firstLine = value.getFirstLineText(1000); + } + const languageSelection = monaco.services.StaticServices.modeService.get().createByFilepathOrFirstLine(uri, firstLine); + this.model = monaco.services.StaticServices.modelService.get().createModel(value, languageSelection, uri); this.resourceVersion = this.resource.version; this.updateSavedVersionId(); this.toDispose.push(this.model); @@ -304,29 +315,30 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument { return; } - const newText = await this.readContents(); - if (newText === undefined || token.isCancellationRequested || this._dirty) { - return; - } - this.resourceVersion = this.resource.version; - - const value = this.model.getValue(); - if (value === newText) { + const value = await this.readContents(); + if (value === undefined || token.isCancellationRequested || this._dirty) { return; } - const range = this.model.getFullModelRange(); - this.applyEdits([{ range, text: newText }], { + this.resourceVersion = this.resource.version; + this.updateModel(() => monaco.services.StaticServices.modelService.get().updateModel(this.model, value), { ignoreDirty: true, ignoreContentChanges: true }); } - protected async readContents(): Promise { + protected async readContents(): Promise { try { - const content = await this.resource.readContents({ encoding: this.getEncoding() }); + const options = { encoding: this.getEncoding() }; + const content = await (this.resource.readStream ? this.resource.readStream(options) : this.resource.readContents(options)); + let value; + if (typeof content === 'string') { + value = content; + } else { + value = monaco.textModel.createTextBufferFactoryFromStream(content); + } this.updateContentEncoding(); this.setValid(true); - return content; + return value; } catch (e) { this.setValid(false); if (ResourceError.NotFound.is(e)) { @@ -404,6 +416,10 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument { operations: monaco.editor.IIdentifiedSingleEditOperation[], options?: Partial ): monaco.editor.IIdentifiedSingleEditOperation[] { + return this.updateModel(() => this.model.applyEdits(operations), options); + } + + protected updateModel(doUpdate: () => T, options?: Partial): T { const resolvedOptions: MonacoEditorModel.ApplyEditsOptions = { ignoreDirty: false, ignoreContentChanges: false, @@ -413,7 +429,7 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument { this.ignoreDirtyEdits = resolvedOptions.ignoreDirty; this.ignoreContentChanges = resolvedOptions.ignoreContentChanges; try { - return this.model.applyEdits(operations); + return doUpdate(); } finally { this.ignoreDirtyEdits = ignoreDirtyEdits; this.ignoreContentChanges = ignoreContentChanges; @@ -435,11 +451,12 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument { return; } - const content = this.model.getValue(); + const contentLength = this.model.getValueLength(); + const content = this.model.createSnapshot() || this.model.getValue(); try { const encoding = this.getEncoding(); const version = this.resourceVersion; - await Resource.save(this.resource, { changes, content, options: { encoding, overwriteEncoding, version } }, token); + await Resource.save(this.resource, { changes, content, contentLength, options: { encoding, overwriteEncoding, version } }, token); this.contentChanges.splice(0, changes.length); this.resourceVersion = this.resource.version; this.updateContentEncoding(); diff --git a/packages/monaco/src/browser/monaco-loader.ts b/packages/monaco/src/browser/monaco-loader.ts index 244593a32dec6..a077f8d280850 100644 --- a/packages/monaco/src/browser/monaco-loader.ts +++ b/packages/monaco/src/browser/monaco-loader.ts @@ -78,7 +78,8 @@ export function loadMonaco(vsRequire: any): Promise { 'vs/platform/contextkey/browser/contextKeyService', 'vs/editor/common/model/wordHelper', 'vs/base/common/errors', - 'vs/base/common/path' + 'vs/base/common/path', + 'vs/editor/common/model/textModel' ], (commands: any, actions: any, keybindingsRegistry: any, keybindingResolver: any, resolvedKeybinding: any, keybindingLabels: any, keyCodes: any, mime: any, editorExtensions: any, simpleServices: any, @@ -91,7 +92,7 @@ export function loadMonaco(vsRequire: any): Promise { markerService: any, contextKey: any, contextKeyService: any, wordHelper: any, - error: any, path: any) => { + error: any, path: any, textModel: any) => { const global: any = self; global.monaco.commands = commands; global.monaco.actions = actions; @@ -114,6 +115,7 @@ export function loadMonaco(vsRequire: any): Promise { global.monaco.wordHelper = wordHelper; global.monaco.error = error; global.monaco.path = path; + global.monaco.textModel = textModel; resolve(); }); }); diff --git a/packages/monaco/src/typings/monaco/index.d.ts b/packages/monaco/src/typings/monaco/index.d.ts index cd6add44bfd48..688d8e5b43914 100644 --- a/packages/monaco/src/typings/monaco/index.d.ts +++ b/packages/monaco/src/typings/monaco/index.d.ts @@ -24,8 +24,24 @@ declare module monaco.instantiation { } } +declare module monaco.textModel { + interface ITextStream { + on(event: 'data', callback: (data: string) => void): void; + on(event: 'error', callback: (err: Error) => void): void; + on(event: 'end', callback: () => void): void; + on(event: string, callback: any): void; + } + // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model/textModel.ts#L58 + export function createTextBufferFactoryFromStream(stream: ITextStream, filter?: (chunk: any) => string, validator?: (chunk: any) => Error | undefined): Promise; +} + declare module monaco.editor { + // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model.ts#L1263 + export interface ITextBufferFactory { + getFirstLineText(lengthLimit: number): string; + } + export interface ICodeEditor { protected readonly _instantiationService: monaco.instantiation.IInstantiationService; @@ -344,6 +360,11 @@ declare module monaco.editor { after?: IContentDecorationRenderOptions; } + // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model.ts#L522 + export interface ITextSnapshot { + read(): string | null; + } + export interface ITextModel { /** * Get the tokens for the line `lineNumber`. @@ -359,6 +380,9 @@ declare module monaco.editor { */ // https://github.com/theia-ide/vscode/blob/standalone/0.19.x/src/vs/editor/common/model.ts#L806-L810 forceTokenization(lineNumber: number): void; + + // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model.ts#L623 + createSnapshot(): ITextSnapshot | null; } } @@ -718,6 +742,12 @@ declare module monaco.services { read(filter?: { owner?: string; resource?: monaco.Uri; severities?: number, take?: number; }): editor.IMarker[]; } + // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/services/modelService.ts#L18 + export interface IModelService { + createModel(value: string | monaco.editor.ITextBufferFactory, languageSelection: ILanguageSelection | null, resource?: monaco.URI, isForSimpleWidget?: boolean): monaco.editor.ITextModel; + updateModel(model: monaco.editor.ITextModel, value: string | monaco.editor.ITextBufferFactory): void; + } + // https://github.com/theia-ide/vscode/blob/standalone/0.19.x/src/vs/editor/standalone/browser/standaloneServices.ts#L56 export module StaticServices { export function init(overrides: monaco.editor.IEditorOverrideServices): [ServiceCollection, monaco.instantiation.IInstantiationService]; @@ -728,6 +758,7 @@ declare module monaco.services { export const resourcePropertiesService: LazyStaticService; export const instantiationService: LazyStaticService; export const markerService: LazyStaticService; + export const modelService: LazyStaticService; } }