From 2a0b6e6db88324d3a7acff35ba3c83b6117455ce Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Thu, 6 Jul 2017 12:22:45 +0200 Subject: [PATCH] Extract AbstractModelLine (#30180) --- src/vs/editor/common/model/modelLine.ts | 594 ++++++++++++------------ 1 file changed, 309 insertions(+), 285 deletions(-) diff --git a/src/vs/editor/common/model/modelLine.ts b/src/vs/editor/common/model/modelLine.ts index b96267c425333..446532f904121 100644 --- a/src/vs/editor/common/model/modelLine.ts +++ b/src/vs/editor/common/model/modelLine.ts @@ -97,12 +97,7 @@ export class MarkersTracker { } } -export interface ITextWithMarkers { - text: string; - markers: LineMarker[]; -} - -interface ITokensAdjuster { +export interface ITokensAdjuster { adjust(toColumn: number, delta: number, minimumAllowedColumn: number): void; finish(delta: number, lineTextLength: number): void; } @@ -188,250 +183,21 @@ export interface IModelLine { split(markersTracker: MarkersTracker, splitColumn: number, forceMoveMarkers: boolean, tabSize: number): ModelLine; } -export class ModelLine implements IModelLine { - - private _text: string; - public get text(): string { return this._text; } - - /** - * bits 31 - 1 => indentLevel - * bit 0 => isInvalid - */ - private _metadata: number; - - public isInvalid(): boolean { - return (this._metadata & 0x00000001) ? true : false; - } - - public setIsInvalid(isInvalid: boolean): void { - this._metadata = (this._metadata & 0xfffffffe) | (isInvalid ? 1 : 0); - } - - /** - * Returns: - * - -1 => the line consists of whitespace - * - otherwise => the indent level is returned value - */ - public getIndentLevel(): number { - return ((this._metadata & 0xfffffffe) >> 1) - 1; - } - - private _setPlusOneIndentLevel(value: number): void { - this._metadata = (this._metadata & 0x00000001) | ((value & 0xefffffff) << 1); - } - - public updateTabSize(tabSize: number): void { - if (tabSize === 0) { - // don't care mark - this._metadata = this._metadata & 0x00000001; - } else { - this._setPlusOneIndentLevel(computePlusOneIndentLevel(this._text, tabSize)); - } - } +export abstract class AbstractModelLine { - private _state: IState; - private _lineTokens: ArrayBuffer; private _markers: LineMarker[]; - constructor(text: string, tabSize: number) { - this._metadata = 0; - this._setText(text, tabSize); - this._state = null; - this._lineTokens = null; + constructor() { this._markers = null; } - // --- BEGIN STATE - - public resetTokenizationState(): void { - this._state = null; - this._lineTokens = null; - } - - public setState(state: IState): void { - this._state = state; - } - - public getState(): IState { - return this._state || null; - } - - // --- END STATE - - // --- BEGIN TOKENS - - private static _getDefaultMetadata(topLevelLanguageId: LanguageId): number { - return ( - (topLevelLanguageId << MetadataConsts.LANGUAGEID_OFFSET) - | (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET) - | (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET) - | (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET) - | (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET) - ) >>> 0; - } - - public setTokens(topLevelLanguageId: LanguageId, tokens: Uint32Array): void { - if (!tokens || tokens.length === 0) { - this._lineTokens = null; - return; - } - if (tokens.length === 2) { - // there is one token - if (tokens[0] === 0 && tokens[1] === ModelLine._getDefaultMetadata(topLevelLanguageId)) { - this._lineTokens = null; - return; - } - } - this._lineTokens = tokens.buffer; - } - - public getTokens(topLevelLanguageId: LanguageId): LineTokens { - let rawLineTokens = this._lineTokens; - if (rawLineTokens) { - return new LineTokens(new Uint32Array(rawLineTokens), this._text); - } - - let lineTokens = new Uint32Array(2); - lineTokens[0] = 0; - lineTokens[1] = ModelLine._getDefaultMetadata(topLevelLanguageId); - return new LineTokens(lineTokens, this._text); - } - - // --- END TOKENS - - private _createTokensAdjuster(): ITokensAdjuster { - if (!this._lineTokens) { - // This line does not have real tokens, so there is nothing to adjust - return NO_OP_TOKENS_ADJUSTER; - } - - let lineTokens = new Uint32Array(this._lineTokens); - let tokensLength = (lineTokens.length >>> 1); - let tokenIndex = 0; - let tokenStartOffset = 0; - let removeTokensCount = 0; - - let adjust = (toColumn: number, delta: number, minimumAllowedColumn: number) => { - // console.log(`------------------------------------------------------------------`); - // console.log(`before call: tokenIndex: ${tokenIndex}: ${lineTokens}`); - // console.log(`adjustTokens: ${toColumn} with delta: ${delta} and [${minimumAllowedColumn}]`); - // console.log(`tokenStartOffset: ${tokenStartOffset}`); - let minimumAllowedIndex = minimumAllowedColumn - 1; - - while (tokenStartOffset < toColumn && tokenIndex < tokensLength) { - - if (tokenStartOffset > 0 && delta !== 0) { - // adjust token's `startIndex` by `delta` - let newTokenStartOffset = Math.max(minimumAllowedIndex, tokenStartOffset + delta); - lineTokens[(tokenIndex << 1)] = newTokenStartOffset; - - // console.log(` * adjusted token start offset for token at ${tokenIndex}: ${newTokenStartOffset}`); - - if (delta < 0) { - let tmpTokenIndex = tokenIndex; - while (tmpTokenIndex > 0) { - let prevTokenStartOffset = lineTokens[((tmpTokenIndex - 1) << 1)]; - if (prevTokenStartOffset >= newTokenStartOffset) { - if (prevTokenStartOffset !== Constants.MAX_UINT_32) { - // console.log(` * marking for deletion token at ${tmpTokenIndex - 1}`); - lineTokens[((tmpTokenIndex - 1) << 1)] = Constants.MAX_UINT_32; - removeTokensCount++; - } - tmpTokenIndex--; - } else { - break; - } - } - } - } - - tokenIndex++; - if (tokenIndex < tokensLength) { - tokenStartOffset = lineTokens[(tokenIndex << 1)]; - } - } - // console.log(`after call: tokenIndex: ${tokenIndex}: ${lineTokens}`); - }; - - let finish = (delta: number, lineTextLength: number) => { - adjust(Constants.MAX_SAFE_SMALL_INTEGER, delta, 1); - - // Mark overflowing tokens for deletion & delete marked tokens - this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(removeTokensCount, lineTextLength)); - }; - - return { - adjust: adjust, - finish: finish - }; - } - - private _markOverflowingTokensForDeletion(removeTokensCount: number, lineTextLength: number): number { - if (!this._lineTokens) { - return removeTokensCount; - } - - let lineTokens = new Uint32Array(this._lineTokens); - let tokensLength = (lineTokens.length >>> 1); - - if (removeTokensCount + 1 === tokensLength) { - // no more removing, cannot end up without any tokens for mode transition reasons - return removeTokensCount; - } - - for (let tokenIndex = tokensLength - 1; tokenIndex > 0; tokenIndex--) { - let tokenStartOffset = lineTokens[(tokenIndex << 1)]; - if (tokenStartOffset < lineTextLength) { - // valid token => stop iterating - return removeTokensCount; - } - - // this token now overflows the text => mark it for removal - if (tokenStartOffset !== Constants.MAX_UINT_32) { - // console.log(` * marking for deletion token at ${tokenIndex}`); - lineTokens[(tokenIndex << 1)] = Constants.MAX_UINT_32; - removeTokensCount++; - - if (removeTokensCount + 1 === tokensLength) { - // no more removing, cannot end up without any tokens for mode transition reasons - return removeTokensCount; - } - } - } - - return removeTokensCount; - } - - private _deleteMarkedTokens(removeTokensCount: number): void { - if (removeTokensCount === 0) { - return; - } + /// - let lineTokens = new Uint32Array(this._lineTokens); - let tokensLength = (lineTokens.length >>> 1); - let newTokens = new Uint32Array(((tokensLength - removeTokensCount) << 1)), newTokenIdx = 0; - for (let i = 0; i < tokensLength; i++) { - let startOffset = lineTokens[(i << 1)]; - if (startOffset === Constants.MAX_UINT_32) { - // marked for deletion - continue; - } - let metadata = lineTokens[(i << 1) + 1]; - newTokens[newTokenIdx++] = startOffset; - newTokens[newTokenIdx++] = metadata; - } - this._lineTokens = newTokens.buffer; - } + protected abstract get text(): string; + protected abstract _setText(text: string, tabSize: number); + protected abstract _createTokensAdjuster(): ITokensAdjuster; - private _setText(text: string, tabSize: number): void { - this._text = text; - if (tabSize === 0) { - // don't care mark - this._metadata = this._metadata & 0x00000001; - } else { - this._setPlusOneIndentLevel(computePlusOneIndentLevel(text, tabSize)); - } - } + /// // private _printMarkers(): string { // if (!this._markers) { @@ -537,7 +303,7 @@ export class ModelLine implements IModelLine { public applyEdits(markersTracker: MarkersTracker, edits: ILineEdit[], tabSize: number): number { let deltaColumn = 0; - let resultText = this._text; + let resultText = this.text; let tokensAdjuster = this._createTokensAdjuster(); let markersAdjuster = this._createMarkersAdjuster(markersTracker); @@ -593,8 +359,8 @@ export class ModelLine implements IModelLine { public split(markersTracker: MarkersTracker, splitColumn: number, forceMoveMarkers: boolean, tabSize: number): ModelLine { // console.log('--> split @ ' + splitColumn + '::: ' + this._printMarkers()); - var myText = this._text.substring(0, splitColumn - 1); - var otherText = this._text.substring(splitColumn - 1); + var myText = this.text.substring(0, splitColumn - 1); + var otherText = this.text.substring(splitColumn - 1); var otherMarkers: LineMarker[] = null; @@ -631,9 +397,6 @@ export class ModelLine implements IModelLine { this._setText(myText, tabSize); - // Mark overflowing tokens for deletion & delete marked tokens - this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(0, this._text.length)); - var otherLine = new ModelLine(otherText, tabSize); if (otherMarkers) { otherLine.addMarkers(otherMarkers); @@ -644,46 +407,18 @@ export class ModelLine implements IModelLine { public append(markersTracker: MarkersTracker, myLineNumber: number, other: ModelLine, tabSize: number): void { // console.log('--> append: THIS :: ' + this._printMarkers()); // console.log('--> append: OTHER :: ' + this._printMarkers()); - let thisTextLength = this._text.length; - this._setText(this._text + other._text, tabSize); + let thisTextLength = this.text.length; + this._setText(this.text + other.text, tabSize); - let otherRawTokens = other._lineTokens; - if (otherRawTokens) { - // Other has real tokens + if (other._markers) { + // Other has markers + let otherMarkers = other._markers; - let otherTokens = new Uint32Array(otherRawTokens); + // Adjust other markers + for (let i = 0, len = otherMarkers.length; i < len; i++) { + let marker = otherMarkers[i]; - // Adjust other tokens - if (thisTextLength > 0) { - for (let i = 0, len = (otherTokens.length >>> 1); i < len; i++) { - otherTokens[(i << 1)] = otherTokens[(i << 1)] + thisTextLength; - } - } - - // Append other tokens - let myRawTokens = this._lineTokens; - if (myRawTokens) { - // I have real tokens - let myTokens = new Uint32Array(myRawTokens); - let result = new Uint32Array(myTokens.length + otherTokens.length); - result.set(myTokens, 0); - result.set(otherTokens, myTokens.length); - this._lineTokens = result.buffer; - } else { - // I don't have real tokens - this._lineTokens = otherTokens.buffer; - } - } - - if (other._markers) { - // Other has markers - let otherMarkers = other._markers; - - // Adjust other markers - for (let i = 0, len = otherMarkers.length; i < len; i++) { - let marker = otherMarkers[i]; - - marker.updatePosition(markersTracker, new Position(myLineNumber, marker.position.column + thisTextLength)); + marker.updatePosition(markersTracker, new Position(myLineNumber, marker.position.column + thisTextLength)); } this.addMarkers(otherMarkers); @@ -773,3 +508,292 @@ export class ModelLine implements IModelLine { return undefined; } } + +export class ModelLine extends AbstractModelLine implements IModelLine { + + private _text: string; + public get text(): string { return this._text; } + + /** + * bits 31 - 1 => indentLevel + * bit 0 => isInvalid + */ + private _metadata: number; + + public isInvalid(): boolean { + return (this._metadata & 0x00000001) ? true : false; + } + + public setIsInvalid(isInvalid: boolean): void { + this._metadata = (this._metadata & 0xfffffffe) | (isInvalid ? 1 : 0); + } + + /** + * Returns: + * - -1 => the line consists of whitespace + * - otherwise => the indent level is returned value + */ + public getIndentLevel(): number { + return ((this._metadata & 0xfffffffe) >> 1) - 1; + } + + private _setPlusOneIndentLevel(value: number): void { + this._metadata = (this._metadata & 0x00000001) | ((value & 0xefffffff) << 1); + } + + public updateTabSize(tabSize: number): void { + if (tabSize === 0) { + // don't care mark + this._metadata = this._metadata & 0x00000001; + } else { + this._setPlusOneIndentLevel(computePlusOneIndentLevel(this._text, tabSize)); + } + } + + private _state: IState; + private _lineTokens: ArrayBuffer; + + constructor(text: string, tabSize: number) { + super(); + this._metadata = 0; + this._setText(text, tabSize); + this._state = null; + this._lineTokens = null; + } + + public split(markersTracker: MarkersTracker, splitColumn: number, forceMoveMarkers: boolean, tabSize: number): ModelLine { + let result = super.split(markersTracker, splitColumn, forceMoveMarkers, tabSize); + + // Mark overflowing tokens for deletion & delete marked tokens + this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(0, this.text.length)); + + return result; + } + + public append(markersTracker: MarkersTracker, myLineNumber: number, other: ModelLine, tabSize: number): void { + let thisTextLength = this.text.length; + + super.append(markersTracker, myLineNumber, other, tabSize); + + let otherRawTokens = other._lineTokens; + if (otherRawTokens) { + // Other has real tokens + + let otherTokens = new Uint32Array(otherRawTokens); + + // Adjust other tokens + if (thisTextLength > 0) { + for (let i = 0, len = (otherTokens.length >>> 1); i < len; i++) { + otherTokens[(i << 1)] = otherTokens[(i << 1)] + thisTextLength; + } + } + + // Append other tokens + let myRawTokens = this._lineTokens; + if (myRawTokens) { + // I have real tokens + let myTokens = new Uint32Array(myRawTokens); + let result = new Uint32Array(myTokens.length + otherTokens.length); + result.set(myTokens, 0); + result.set(otherTokens, myTokens.length); + this._lineTokens = result.buffer; + } else { + // I don't have real tokens + this._lineTokens = otherTokens.buffer; + } + } + } + + // --- BEGIN STATE + + public resetTokenizationState(): void { + this._state = null; + this._lineTokens = null; + } + + public setState(state: IState): void { + this._state = state; + } + + public getState(): IState { + return this._state || null; + } + + // --- END STATE + + // --- BEGIN TOKENS + + private static _getDefaultMetadata(topLevelLanguageId: LanguageId): number { + return ( + (topLevelLanguageId << MetadataConsts.LANGUAGEID_OFFSET) + | (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET) + | (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET) + | (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET) + | (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET) + ) >>> 0; + } + + public setTokens(topLevelLanguageId: LanguageId, tokens: Uint32Array): void { + if (!tokens || tokens.length === 0) { + this._lineTokens = null; + return; + } + if (tokens.length === 2) { + // there is one token + if (tokens[0] === 0 && tokens[1] === ModelLine._getDefaultMetadata(topLevelLanguageId)) { + this._lineTokens = null; + return; + } + } + this._lineTokens = tokens.buffer; + } + + public getTokens(topLevelLanguageId: LanguageId): LineTokens { + let rawLineTokens = this._lineTokens; + if (rawLineTokens) { + return new LineTokens(new Uint32Array(rawLineTokens), this._text); + } + + let lineTokens = new Uint32Array(2); + lineTokens[0] = 0; + lineTokens[1] = ModelLine._getDefaultMetadata(topLevelLanguageId); + return new LineTokens(lineTokens, this._text); + } + + // --- END TOKENS + + protected _createTokensAdjuster(): ITokensAdjuster { + if (!this._lineTokens) { + // This line does not have real tokens, so there is nothing to adjust + return NO_OP_TOKENS_ADJUSTER; + } + + let lineTokens = new Uint32Array(this._lineTokens); + let tokensLength = (lineTokens.length >>> 1); + let tokenIndex = 0; + let tokenStartOffset = 0; + let removeTokensCount = 0; + + let adjust = (toColumn: number, delta: number, minimumAllowedColumn: number) => { + // console.log(`------------------------------------------------------------------`); + // console.log(`before call: tokenIndex: ${tokenIndex}: ${lineTokens}`); + // console.log(`adjustTokens: ${toColumn} with delta: ${delta} and [${minimumAllowedColumn}]`); + // console.log(`tokenStartOffset: ${tokenStartOffset}`); + let minimumAllowedIndex = minimumAllowedColumn - 1; + + while (tokenStartOffset < toColumn && tokenIndex < tokensLength) { + + if (tokenStartOffset > 0 && delta !== 0) { + // adjust token's `startIndex` by `delta` + let newTokenStartOffset = Math.max(minimumAllowedIndex, tokenStartOffset + delta); + lineTokens[(tokenIndex << 1)] = newTokenStartOffset; + + // console.log(` * adjusted token start offset for token at ${tokenIndex}: ${newTokenStartOffset}`); + + if (delta < 0) { + let tmpTokenIndex = tokenIndex; + while (tmpTokenIndex > 0) { + let prevTokenStartOffset = lineTokens[((tmpTokenIndex - 1) << 1)]; + if (prevTokenStartOffset >= newTokenStartOffset) { + if (prevTokenStartOffset !== Constants.MAX_UINT_32) { + // console.log(` * marking for deletion token at ${tmpTokenIndex - 1}`); + lineTokens[((tmpTokenIndex - 1) << 1)] = Constants.MAX_UINT_32; + removeTokensCount++; + } + tmpTokenIndex--; + } else { + break; + } + } + } + } + + tokenIndex++; + if (tokenIndex < tokensLength) { + tokenStartOffset = lineTokens[(tokenIndex << 1)]; + } + } + // console.log(`after call: tokenIndex: ${tokenIndex}: ${lineTokens}`); + }; + + let finish = (delta: number, lineTextLength: number) => { + adjust(Constants.MAX_SAFE_SMALL_INTEGER, delta, 1); + + // Mark overflowing tokens for deletion & delete marked tokens + this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(removeTokensCount, lineTextLength)); + }; + + return { + adjust: adjust, + finish: finish + }; + } + + private _markOverflowingTokensForDeletion(removeTokensCount: number, lineTextLength: number): number { + if (!this._lineTokens) { + return removeTokensCount; + } + + let lineTokens = new Uint32Array(this._lineTokens); + let tokensLength = (lineTokens.length >>> 1); + + if (removeTokensCount + 1 === tokensLength) { + // no more removing, cannot end up without any tokens for mode transition reasons + return removeTokensCount; + } + + for (let tokenIndex = tokensLength - 1; tokenIndex > 0; tokenIndex--) { + let tokenStartOffset = lineTokens[(tokenIndex << 1)]; + if (tokenStartOffset < lineTextLength) { + // valid token => stop iterating + return removeTokensCount; + } + + // this token now overflows the text => mark it for removal + if (tokenStartOffset !== Constants.MAX_UINT_32) { + // console.log(` * marking for deletion token at ${tokenIndex}`); + lineTokens[(tokenIndex << 1)] = Constants.MAX_UINT_32; + removeTokensCount++; + + if (removeTokensCount + 1 === tokensLength) { + // no more removing, cannot end up without any tokens for mode transition reasons + return removeTokensCount; + } + } + } + + return removeTokensCount; + } + + private _deleteMarkedTokens(removeTokensCount: number): void { + if (removeTokensCount === 0) { + return; + } + + let lineTokens = new Uint32Array(this._lineTokens); + let tokensLength = (lineTokens.length >>> 1); + let newTokens = new Uint32Array(((tokensLength - removeTokensCount) << 1)), newTokenIdx = 0; + for (let i = 0; i < tokensLength; i++) { + let startOffset = lineTokens[(i << 1)]; + if (startOffset === Constants.MAX_UINT_32) { + // marked for deletion + continue; + } + let metadata = lineTokens[(i << 1) + 1]; + newTokens[newTokenIdx++] = startOffset; + newTokens[newTokenIdx++] = metadata; + } + this._lineTokens = newTokens.buffer; + } + + protected _setText(text: string, tabSize: number): void { + this._text = text; + if (tabSize === 0) { + // don't care mark + this._metadata = this._metadata & 0x00000001; + } else { + this._setPlusOneIndentLevel(computePlusOneIndentLevel(text, tabSize)); + } + } + +}