[monaco] stream content to avoid blocking the backend and network by …

…large files

Signed-off-by: Anton Kosyakov <[email protected]>

packages/core/src/common/resource.ts

@@ -23,6 +23,7 @@ import { Disposable } from './disposable';
 import { MaybePromise } from './types';
 import { CancellationToken } from './cancellation';
 import { ApplicationError } from './application-error';
+import { ReadableStream, Readable } from './stream';
 
 export interface ResourceVersion {
 }
@@ -62,6 +63,15 @@ export interface Resource extends Disposable {
      * @throws `ResourceError.NotFound` if a resource not found
      */
     readContents(options?: ResourceReadOptions): Promise<string>;
+    /**
+     * Stream latest content of this resource.
+     *
+     * If a resource supports versioning it updates version to latest.
+     * If a resource supports encoding it updates encoding to latest.
+     *
+     * @throws `ResourceError.NotFound` if a resource not found
+     */
+    readStream?(options?: ResourceReadOptions): Promise<ReadableStream<string>>;
     /**
      * Rewrites the complete content for this resource.
      * If a resource does not exist it will be created.
@@ -74,6 +84,18 @@ export interface Resource extends Disposable {
      * @throws `ResourceError.OutOfSync` if latest resource version is out of sync with the given
      */
     saveContents?(content: string, options?: ResourceSaveOptions): Promise<void>;
+    /**
+     * Rewrites the complete content for this resource.
+     * If a resource does not exist it will be created.
+     *
+     * If a resource supports versioning clients can pass some version
+     * to check against it, if it is not provided latest version is used.
+     *
+     * It updates version and encoding to latest.
+     *
+     * @throws `ResourceError.OutOfSync` if latest resource version is out of sync with the given
+     */
+    saveStream?(content: Readable<string>, options?: ResourceSaveOptions): Promise<void>;
     /**
      * Applies incremental content changes to this resource.
      *
@@ -90,7 +112,8 @@ export interface Resource extends Disposable {
 }
 export namespace Resource {
     export interface SaveContext {
-        content: string
+        contentLength: number
+        content: string | Readable<string>
         changes?: TextDocumentContentChangeEvent[]
         options?: ResourceSaveOptions
     }
@@ -104,10 +127,15 @@ export namespace Resource {
         if (token && token.isCancellationRequested) {
             return;
         }
-        await resource.saveContents(context.content, context.options);
+        if (typeof context.content !== 'string' && resource.saveStream) {
+            await resource.saveStream(context.content, context.options);
+        } else {
+            const content = typeof context.content === 'string' ? context.content : Readable.toString(context.content);
+            await resource.saveContents(content, context.options);
+        }
     }
     export async function trySaveContentChanges(resource: Resource, context: SaveContext): Promise<boolean> {
-        if (!context.changes || !resource.saveContentChanges || shouldSaveContent(context)) {
+        if (!context.changes || !resource.saveContentChanges || shouldSaveContent(resource, context)) {
             return false;
         }
         try {
@@ -120,12 +148,11 @@ export namespace Resource {
             return false;
         }
     }
-    export function shouldSaveContent({ content, changes }: SaveContext): boolean {
-        if (!changes) {
+    export function shouldSaveContent(resource: Resource, { contentLength, changes }: SaveContext): boolean {
+        if (!changes || (resource.saveStream && contentLength > 32 * 1024 * 1024)) {
             return true;
         }
         let contentChangesLength = 0;
-        const contentLength = content.length;
         for (const change of changes) {
             contentChangesLength += JSON.stringify(change).length;
             if (contentChangesLength > contentLength) {

packages/core/src/common/stream.ts

@@ -103,6 +103,14 @@ export namespace Readable {
             }
         };
     }
+    export function toString(readable: Readable<string>): string {
+        let result = '';
+        let chunk: string | null;
+        while ((chunk = readable.read()) != null) {
+            result += chunk;
+        }
+        return result;
+    }
 }
 
 /**

packages/filesystem/src/browser/file-resource.ts

@@ -18,6 +18,7 @@ import { injectable, inject } from 'inversify';
 import { Resource, ResourceVersion, ResourceResolver, ResourceError, ResourceSaveOptions } from '@theia/core/lib/common/resource';
 import { DisposableCollection } from '@theia/core/lib/common/disposable';
 import { Emitter, Event } from '@theia/core/lib/common/event';
+import { Readable, ReadableStream } from '@theia/core/lib/common/stream';
 import URI from '@theia/core/lib/common/uri';
 import { FileOperation, FileOperationError, FileOperationResult, ETAG_DISABLED, FileSystemProviderCapabilities, FileReadStreamOptions, BinarySize } from '../common/files';
 import { FileService, TextFileOperationError, TextFileOperationResult } from './file-service';
@@ -135,7 +136,59 @@ export class FileResource implements Resource {
         }
     }
 
-    async saveContents(content: string, options?: ResourceSaveOptions): Promise<void> {
+    async readStream(options?: { encoding?: string }): Promise<ReadableStream<string>> {
+        try {
+            const encoding = options?.encoding || this.version?.encoding;
+            const stat = await this.fileService.readStream(this.uri, {
+                encoding,
+                etag: ETAG_DISABLED,
+                acceptTextOnly: this.acceptTextOnly,
+                limits: this.limits
+            });
+            this._version = {
+                encoding: stat.encoding,
+                etag: stat.etag,
+                mtime: stat.mtime
+            };
+            return stat.value;
+        } catch (e) {
+            if (e instanceof TextFileOperationError && e.textFileOperationResult === TextFileOperationResult.FILE_IS_BINARY) {
+                if (await this.shouldOpenAsText('The file is either binary or uses an unsupported text encoding.')) {
+                    this.acceptTextOnly = false;
+                    return this.readStream(options);
+                }
+            } else if (e instanceof FileOperationError && e.fileOperationResult === FileOperationResult.FILE_TOO_LARGE) {
+                const stat = await this.fileService.resolve(this.uri, { resolveMetadata: true });
+                const maxFileSize = GENERAL_MAX_FILE_SIZE_MB * 1024 * 1024;
+                if (this.limits?.size !== maxFileSize && await this.shouldOpenAsText(`The file is too large (${BinarySize.formatSize(stat.size)}).`)) {
+                    this.limits = {
+                        size: maxFileSize
+                    };
+                    return this.readStream(options);
+                }
+            } else if (e instanceof FileOperationError && e.fileOperationResult === FileOperationResult.FILE_NOT_FOUND) {
+                this._version = undefined;
+                const { message, stack } = e;
+                throw ResourceError.NotFound({
+                    message, stack,
+                    data: {
+                        uri: this.uri
+                    }
+                });
+            }
+            throw e;
+        }
+    }
+
+    saveContents(content: string, options?: ResourceSaveOptions): Promise<void> {
+        return this.doWrite(content, options);
+    }
+
+    saveStream(content: Readable<string>, options?: ResourceSaveOptions): Promise<void> {
+        return this.doWrite(content, options);
+    }
+
+    protected async doWrite(content: string | Readable<string>, options?: ResourceSaveOptions): Promise<void> {
         const version = options?.version || this._version;
         const current = FileResourceVersion.is(version) ? version : undefined;
         const etag = current?.etag;
@@ -154,7 +207,7 @@ export class FileResource implements Resource {
         } catch (e) {
             if (e instanceof FileOperationError && e.fileOperationResult === FileOperationResult.FILE_MODIFIED_SINCE) {
                 if (etag !== ETAG_DISABLED && await this.shouldOverwrite()) {
-                    return this.saveContents(content, { ...options, version: { stat: { ...current, etag: ETAG_DISABLED } } });
+                    return this.doWrite(content, { ...options, version: { stat: { ...current, etag: ETAG_DISABLED } } });
                 }
                 const { message, stack } = e;
                 throw ResourceError.OutOfSync({ message, stack, data: { uri: this.uri } });

packages/monaco/src/browser/monaco-editor-model.ts

@@ -150,9 +150,21 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
      * Only this method can create an instance of `monaco.editor.IModel`,
      * there should not be other calls to `monaco.editor.createModel`.
      */
-    protected initialize(content: string): void {
+    protected initialize(value: string | monaco.editor.ITextBufferFactory): void {
         if (!this.toDispose.disposed) {
-            this.model = monaco.editor.createModel(content, undefined, monaco.Uri.parse(this.resource.uri.toString()));
+            const uri = monaco.Uri.parse(this.resource.uri.toString());
+            let firstLine;
+            if (typeof value === 'string') {
+                firstLine = value;
+                const firstLF = value.indexOf('\n');
+                if (firstLF !== -1) {
+                    firstLine = value.substring(0, firstLF);
+                }
+            } else {
+                firstLine = value.getFirstLineText(1000);
+            }
+            const languageSelection = monaco.services.StaticServices.modeService.get().createByFilepathOrFirstLine(uri, firstLine);
+            this.model = monaco.services.StaticServices.modelService.get().createModel(value, languageSelection, uri);
             this.resourceVersion = this.resource.version;
             this.updateSavedVersionId();
             this.toDispose.push(this.model);
@@ -311,29 +323,30 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
             return;
         }
 
-        const newText = await this.readContents();
-        if (newText === undefined || token.isCancellationRequested || this._dirty) {
-            return;
-        }
-        this.resourceVersion = this.resource.version;
-
-        const value = this.model.getValue();
-        if (value === newText) {
+        const value = await this.readContents();
+        if (value === undefined || token.isCancellationRequested || this._dirty) {
             return;
         }
 
-        const range = this.m2p.asRange(this.model.getFullModelRange());
-        this.applyEdits([this.p2m.asTextEdit({ range, newText }) as monaco.editor.IIdentifiedSingleEditOperation], {
+        this.resourceVersion = this.resource.version;
+        this.updateModel(() => monaco.services.StaticServices.modelService.get().updateModel(this.model, value), {
             ignoreDirty: true,
             ignoreContentChanges: true
         });
     }
-    protected async readContents(): Promise<string | undefined> {
+    protected async readContents(): Promise<string | monaco.editor.ITextBufferFactory | undefined> {
         try {
-            const content = await this.resource.readContents({ encoding: this.getEncoding() });
+            const options = { encoding: this.getEncoding() };
+            const content = await (this.resource.readStream ? this.resource.readStream(options) : this.resource.readContents(options));
+            let value;
+            if (typeof content === 'string') {
+                value = content;
+            } else {
+                value = monaco.textModel.createTextBufferFactoryFromStream(content);
+            }
             this.updateContentEncoding();
             this.setValid(true);
-            return content;
+            return value;
         } catch (e) {
             this.setValid(false);
             if (ResourceError.NotFound.is(e)) {
@@ -411,6 +424,10 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
         operations: monaco.editor.IIdentifiedSingleEditOperation[],
         options?: Partial<MonacoEditorModel.ApplyEditsOptions>
     ): monaco.editor.IIdentifiedSingleEditOperation[] {
+        return this.updateModel(() => this.model.applyEdits(operations), options);
+    }
+
+    protected updateModel<T>(doUpdate: () => T, options?: Partial<MonacoEditorModel.ApplyEditsOptions>): T {
         const resolvedOptions: MonacoEditorModel.ApplyEditsOptions = {
             ignoreDirty: false,
             ignoreContentChanges: false,
@@ -420,7 +437,7 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
         this.ignoreDirtyEdits = resolvedOptions.ignoreDirty;
         this.ignoreContentChanges = resolvedOptions.ignoreContentChanges;
         try {
-            return this.model.applyEdits(operations);
+            return doUpdate();
         } finally {
             this.ignoreDirtyEdits = ignoreDirtyEdits;
             this.ignoreContentChanges = ignoreContentChanges;
@@ -442,11 +459,12 @@ export class MonacoEditorModel implements ITextEditorModel, TextEditorDocument {
             return;
         }
 
-        const content = this.model.getValue();
+        const contentLength = this.model.getValueLength();
+        const content = this.model.createSnapshot() || this.model.getValue();
         try {
             const encoding = this.getEncoding();
             const version = this.resourceVersion;
-            await Resource.save(this.resource, { changes, content, options: { encoding, overwriteEncoding, version } }, token);
+            await Resource.save(this.resource, { changes, content, contentLength, options: { encoding, overwriteEncoding, version } }, token);
             this.contentChanges.splice(0, changes.length);
             this.resourceVersion = this.resource.version;
             this.updateContentEncoding();

packages/monaco/src/browser/monaco-loader.ts

@@ -76,7 +76,8 @@ export function loadMonaco(vsRequire: any): Promise<void> {
                 'vs/platform/contextkey/common/contextkey',
                 'vs/platform/contextkey/browser/contextKeyService',
                 'vs/editor/common/model/wordHelper',
-                'vs/base/common/errors'
+                'vs/base/common/errors',
+                'vs/editor/common/model/textModel',
             ], (commands: any, actions: any,
                 keybindingsRegistry: any, keybindingResolver: any, resolvedKeybinding: any, keybindingLabels: any,
                 keyCodes: any, mime: any, editorExtensions: any, simpleServices: any,
@@ -89,7 +90,7 @@ export function loadMonaco(vsRequire: any): Promise<void> {
                 markerService: any,
                 contextKey: any, contextKeyService: any,
                 wordHelper: any,
-                error: any) => {
+                error: any, textModel: any) => {
                 const global: any = self;
                 global.monaco.commands = commands;
                 global.monaco.actions = actions;
@@ -111,6 +112,7 @@ export function loadMonaco(vsRequire: any): Promise<void> {
                 global.monaco.mime = mime;
                 global.monaco.wordHelper = wordHelper;
                 global.monaco.error = error;
+                global.monaco.textModel = textModel;
                 resolve();
             });
         });

packages/monaco/src/typings/monaco/index.d.ts

@@ -24,8 +24,24 @@ declare module monaco.instantiation {
     }
 }
 
+declare module monaco.textModel {
+    interface ITextStream {
+        on(event: 'data', callback: (data: string) => void): void;
+        on(event: 'error', callback: (err: Error) => void): void;
+        on(event: 'end', callback: () => void): void;
+        on(event: string, callback: any): void;
+    }
+    // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model/textModel.ts#L58
+    export function createTextBufferFactoryFromStream(stream: ITextStream, filter?: (chunk: any) => string, validator?: (chunk: any) => Error | undefined): Promise<monaco.editor.ITextBufferFactory>;
+}
+
 declare module monaco.editor {
 
+    // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model.ts#L1263
+    export interface ITextBufferFactory {
+        getFirstLineText(lengthLimit: number): string;
+    }
+
     export interface ICodeEditor {
         protected readonly _instantiationService: monaco.instantiation.IInstantiationService;
 
@@ -344,6 +360,11 @@ declare module monaco.editor {
         after?: IContentDecorationRenderOptions;
     }
 
+    // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model.ts#L522
+    export interface ITextSnapshot {
+        read(): string | null;
+    }
+
     export interface ITextModel {
         /**
          * Get the tokens for the line `lineNumber`.
@@ -359,6 +380,9 @@ declare module monaco.editor {
          */
         // https://github.com/theia-ide/vscode/blob/standalone/0.19.x/src/vs/editor/common/model.ts#L806-L810
         forceTokenization(lineNumber: number): void;
+
+        // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/model.ts#L623
+        createSnapshot(): ITextSnapshot | null;
     }
 
 }
@@ -718,6 +742,12 @@ declare module monaco.services {
         read(filter?: { owner?: string; resource?: monaco.Uri; severities?: number, take?: number; }): editor.IMarker[];
     }
 
+    // https://github.com/microsoft/vscode/blob/e683ace9e5acadba0e8bde72d793cb2cb83e58a7/src/vs/editor/common/services/modelService.ts#L18
+    export interface IModelService {
+        createModel(value: string | monaco.editor.ITextBufferFactory, languageSelection: ILanguageSelection | null, resource?: monaco.URI, isForSimpleWidget?: boolean): monaco.editor.ITextModel;
+        updateModel(model: monaco.editor.ITextModel, value: string | monaco.editor.ITextBufferFactory): void;
+    }
+
     // https://github.com/theia-ide/vscode/blob/standalone/0.19.x/src/vs/editor/standalone/browser/standaloneServices.ts#L56
     export module StaticServices {
         export function init(overrides: monaco.editor.IEditorOverrideServices): [ServiceCollection, monaco.instantiation.IInstantiationService];
@@ -728,6 +758,7 @@ declare module monaco.services {
         export const resourcePropertiesService: LazyStaticService<ITextResourcePropertiesService>;
         export const instantiationService: LazyStaticService<monaco.instantiation.IInstantiationService>;
         export const markerService: LazyStaticService<IMarkerService>;
+        export const modelService: LazyStaticService<IModelService>;
     }
 }