Skip to content

Commit

Permalink
Introduce and begin adopting StandardTokenType
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexdima
alexdima committed Nov 1, 2016
1 parent 3be9dad commit b70ce79
Show file tree
Hide file tree
Showing 6 changed files with 95 additions and 45 deletions.
33 changes: 33 additions & 0 deletions src/vs/editor/common/core/lineTokens.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,33 @@ import { TokensBinaryEncoding, TokensInflatorMap } from 'vs/editor/common/model/
import { ModeTransition } from 'vs/editor/common/core/modeTransition';
import { ViewLineToken } from 'vs/editor/common/core/viewLineToken';

/**
* A standard token type.
*/
export const enum StandardTokenType {
Other = 0,
Comment = 1,
String = 2,
RegEx = 3
}

const STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex)\b/;
function toStandardTokenType(tokenType: string): StandardTokenType {
let m = tokenType.match(STANDARD_TOKEN_TYPE_REGEXP);
if (!m) {
return StandardTokenType.Other;
}
switch (m[1]) {
case 'comment':
return StandardTokenType.Comment;
case 'string':
return StandardTokenType.String;
case 'regex':
return StandardTokenType.RegEx;
}
throw new Error('Unexpected match for standard token type!');
}

export class LineToken {
_lineTokenBrand: void;

Expand All @@ -18,6 +45,7 @@ export class LineToken {
public readonly startOffset: number;
public readonly endOffset: number;
public readonly type: string;
public readonly standardType: StandardTokenType;
public readonly modeId: string;
public readonly hasPrev: boolean;
public readonly hasNext: boolean;
Expand All @@ -30,6 +58,7 @@ export class LineToken {
this.startOffset = this._source.getTokenStartOffset(this._tokenIndex);
this.endOffset = this._source.getTokenEndOffset(this._tokenIndex);
this.type = this._source.getTokenType(this._tokenIndex);
this.standardType = toStandardTokenType(this.type);
this.modeId = this._source.modeTransitions[this._modeIndex].modeId;
this.hasPrev = (this._tokenIndex > 0);
this.hasNext = (this._tokenIndex + 1 < this._source.getTokenCount());
Expand Down Expand Up @@ -106,6 +135,10 @@ export class LineTokens {
return TokensBinaryEncoding.getType(this._map, this._tokens[tokenIndex]);
}

public getStandardTokenType(tokenIndex: number): StandardTokenType {
return toStandardTokenType(this.getTokenType(tokenIndex));
}

public getTokenEndOffset(tokenIndex: number): number {
if (tokenIndex + 1 < this._tokens.length) {
return TokensBinaryEncoding.getStartIndex(this._tokens[tokenIndex + 1]);
Expand Down
20 changes: 10 additions & 10 deletions src/vs/editor/common/model/textModelWithTokens.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { TokenIterator } from 'vs/editor/common/model/tokenIterator';
import { ITokenizationSupport, ILineTokens, IMode, IState, TokenizationRegistry } from 'vs/editor/common/modes';
import { NULL_MODE_ID, nullTokenize } from 'vs/editor/common/modes/nullMode';
import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
import { BracketsUtils, RichEditBrackets, IRichEditBracket } from 'vs/editor/common/modes/supports/richEditBrackets';
import { BracketsUtils, RichEditBrackets, RichEditBracket } from 'vs/editor/common/modes/supports/richEditBrackets';
import { ModeTransition } from 'vs/editor/common/core/modeTransition';
import { TokensInflatorMap } from 'vs/editor/common/model/tokensBinaryEncoding';
import { Position } from 'vs/editor/common/core/position';
Expand Down Expand Up @@ -548,7 +548,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
const currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.modeId);

// check that the token is not to be ignored
if (currentModeBrackets && !ignoreBracketsInToken(currentToken.type)) {
if (currentModeBrackets && !ignoreBracketsInToken(currentToken.standardType)) {
// limit search to not go before `maxBracketLength`
let searchStartOffset = Math.max(currentToken.startOffset, position.column - 1 - currentModeBrackets.maxBracketLength);
// limit search to not go after `maxBracketLength`
Expand Down Expand Up @@ -599,7 +599,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
const prevModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(prevToken.modeId);

// check that previous token is not to be ignored
if (prevModeBrackets && !ignoreBracketsInToken(prevToken.type)) {
if (prevModeBrackets && !ignoreBracketsInToken(prevToken.standardType)) {
// limit search in case previous token is very large, there's no need to go beyond `maxBracketLength`
const searchStartOffset = Math.max(prevToken.startOffset, position.column - 1 - prevModeBrackets.maxBracketLength);
const searchEndOffset = currentToken.startOffset;
Expand All @@ -623,7 +623,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
return null;
}

private _matchFoundBracket(foundBracket: Range, data: IRichEditBracket, isOpen: boolean): [Range, Range] {
private _matchFoundBracket(foundBracket: Range, data: RichEditBracket, isOpen: boolean): [Range, Range] {
if (isOpen) {
let matched = this._findMatchingBracketDown(data, foundBracket.getEndPosition());
if (matched) {
Expand All @@ -639,7 +639,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
return null;
}

private _findMatchingBracketUp(bracket: IRichEditBracket, position: Position): Range {
private _findMatchingBracketUp(bracket: RichEditBracket, position: Position): Range {
// console.log('_findMatchingBracketUp: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));

const modeId = bracket.modeId;
Expand All @@ -663,7 +663,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
}

while (currentToken) {
if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.standardType)) {

while (true) {
let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset);
Expand Down Expand Up @@ -698,7 +698,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
return null;
}

private _findMatchingBracketDown(bracket: IRichEditBracket, position: Position): Range {
private _findMatchingBracketDown(bracket: RichEditBracket, position: Position): Range {
// console.log('_findMatchingBracketDown: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));

const modeId = bracket.modeId;
Expand All @@ -722,7 +722,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
}

while (currentToken) {
if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.standardType)) {
while (true) {
let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
if (!r) {
Expand Down Expand Up @@ -782,7 +782,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
modeId = currentToken.modeId;
modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(modeId);
}
if (modeBrackets && !ignoreBracketsInToken(currentToken.type)) {
if (modeBrackets && !ignoreBracketsInToken(currentToken.standardType)) {
let r = BracketsUtils.findPrevBracketInToken(modeBrackets.reversedRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset);
if (r) {
return this._toFoundBracket(modeBrackets, r);
Expand Down Expand Up @@ -825,7 +825,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
modeId = currentToken.modeId;
modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(modeId);
}
if (modeBrackets && !ignoreBracketsInToken(currentToken.type)) {
if (modeBrackets && !ignoreBracketsInToken(currentToken.standardType)) {
let r = BracketsUtils.findNextBracketInToken(modeBrackets.forwardRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
if (r) {
return this._toFoundBracket(modeBrackets, r);
Expand Down
15 changes: 11 additions & 4 deletions src/vs/editor/common/modes/supports.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { TPromise } from 'vs/base/common/winjs.base';
import * as modes from 'vs/editor/common/modes';
import { ModeTransition } from 'vs/editor/common/core/modeTransition';
import { Token } from 'vs/editor/common/core/token';
import { LineTokens } from 'vs/editor/common/core/lineTokens';
import { LineTokens, StandardTokenType } from 'vs/editor/common/core/lineTokens';

export class RawLineTokens implements modes.ILineTokens {
_lineTokensBrand: void;
Expand Down Expand Up @@ -99,9 +99,16 @@ export class ScopedLineTokens {
public getTokenType(tokenIndex: number): string {
return this._actual.getTokenType(tokenIndex + this._firstTokenIndex);
}

public getStandardTokenType(tokenIndex: number): StandardTokenType {
return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex);
}
}

const IGNORE_IN_TOKENS = /\b(comment|string|regex)\b/;
export function ignoreBracketsInToken(tokenType: string): boolean {
return IGNORE_IN_TOKENS.test(tokenType);
export function ignoreBracketsInToken(standardTokenType: StandardTokenType): boolean {
return (
standardTokenType === StandardTokenType.Comment
|| standardTokenType === StandardTokenType.String
|| standardTokenType === StandardTokenType.RegEx
);
}
2 changes: 1 addition & 1 deletion src/vs/editor/common/modes/supports/electricCharacter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ export class BracketElectricCharacterSupport {
return null;
}

if (!ignoreBracketsInToken(context.getTokenType(tokenIndex))) {
if (!ignoreBracketsInToken(context.getStandardTokenType(tokenIndex))) {
let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, 1, lineText, tokenStart, tokenEnd);
if (r) {
let text = lineText.substring(r.startColumn - 1, r.endColumn - 1);
Expand Down
30 changes: 20 additions & 10 deletions src/vs/editor/common/modes/supports/richEditBrackets.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,33 +13,43 @@ interface ISimpleInternalBracket {
close: string;
}

export interface IRichEditBracket {
export class RichEditBracket {
_richEditBracketBrand: void;

readonly modeId: string;
readonly open: string;
readonly close: string;
readonly forwardRegex: RegExp;
readonly reversedRegex: RegExp;

constructor(modeId: string, open: string, close: string, forwardRegex: RegExp, reversedRegex: RegExp) {
this.modeId = modeId;
this.open = open;
this.close = close;
this.forwardRegex = forwardRegex;
this.reversedRegex = reversedRegex;
}
}

export class RichEditBrackets {
_richEditBracketsBrand: void;

public readonly brackets: IRichEditBracket[];
public readonly brackets: RichEditBracket[];
public readonly forwardRegex: RegExp;
public readonly reversedRegex: RegExp;
public readonly maxBracketLength: number;
public readonly textIsBracket: { [text: string]: IRichEditBracket; };
public readonly textIsBracket: { [text: string]: RichEditBracket; };
public readonly textIsOpenBracket: { [text: string]: boolean; };

constructor(modeId: string, brackets: CharacterPair[]) {
this.brackets = brackets.map((b) => {
return {
modeId: modeId,
open: b[0],
close: b[1],
forwardRegex: getRegexForBracketPair({ open: b[0], close: b[1] }),
reversedRegex: getReversedRegexForBracketPair({ open: b[0], close: b[1] })
};
return new RichEditBracket(
modeId,
b[0],
b[1],
getRegexForBracketPair({ open: b[0], close: b[1] }),
getReversedRegexForBracketPair({ open: b[0], close: b[1] })
);
});
this.forwardRegex = getRegexForBrackets(this.brackets);
this.reversedRegex = getReversedRegexForBrackets(this.brackets);
Expand Down
40 changes: 20 additions & 20 deletions src/vs/editor/contrib/smartSelect/common/tokenTree.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import { Position } from 'vs/editor/common/core/position';
import { Range } from 'vs/editor/common/core/range';
import { IModel, IPosition } from 'vs/editor/common/editorCommon';
import { LineToken } from 'vs/editor/common/core/lineTokens';
import { LineToken, StandardTokenType } from 'vs/editor/common/core/lineTokens';
import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
import { BracketsUtils, RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
Expand Down Expand Up @@ -103,14 +103,14 @@ export class Block extends Node {
class Token {
_tokenBrand: void;

range: Range;
type: string;
bracket: TokenTreeBracket;
readonly range: Range;
readonly bracket: TokenTreeBracket;
readonly bracketType: string;

constructor(range: Range, type: string, bracket: TokenTreeBracket) {
constructor(range: Range, bracket: TokenTreeBracket, bracketType: string) {
this.range = range;
this.type = type;
this.bracket = bracket;
this.bracketType = bracketType;
}
}

Expand All @@ -128,15 +128,15 @@ class RawToken {
public lineText: string;
public startOffset: number;
public endOffset: number;
public type: string;
public standardType: StandardTokenType;
public modeId: string;

constructor(source: LineToken, lineNumber: number, lineText: string) {
this.lineNumber = lineNumber;
this.lineText = lineText;
this.startOffset = source.startOffset;
this.endOffset = source.endOffset;
this.type = source.type;
this.standardType = source.standardType;
this.modeId = source.modeId;
}
}
Expand Down Expand Up @@ -209,7 +209,7 @@ class TokenScanner {
}
const lineNumber = token.lineNumber;
const lineText = token.lineText;
const tokenType = token.type;
const standardTokenType = token.standardType;
let startOffset = token.startOffset;
const endOffset = token.endOffset;

Expand All @@ -219,11 +219,11 @@ class TokenScanner {
}
const modeBrackets = this._cachedModeBrackets;

if (!modeBrackets || ignoreBracketsInToken(tokenType)) {
if (!modeBrackets || ignoreBracketsInToken(standardTokenType)) {
return new Token(
new Range(lineNumber, startOffset + 1, lineNumber, endOffset + 1),
tokenType,
TokenTreeBracket.None
TokenTreeBracket.None,
null
);
}

Expand All @@ -238,8 +238,8 @@ class TokenScanner {
// there is some text before this bracket in this token
this._nextBuff.push(new Token(
new Range(lineNumber, startOffset + 1, lineNumber, foundBracketStartOffset + 1),
tokenType,
TokenTreeBracket.None
TokenTreeBracket.None,
null
));
}

Expand All @@ -251,8 +251,8 @@ class TokenScanner {

this._nextBuff.push(new Token(
new Range(lineNumber, foundBracketStartOffset + 1, lineNumber, foundBracketEndOffset + 1),
`${bracketData.modeId};${bracketData.open};${bracketData.close}`,
bracketIsOpen ? TokenTreeBracket.Open : TokenTreeBracket.Close
bracketIsOpen ? TokenTreeBracket.Open : TokenTreeBracket.Close,
`${bracketData.modeId};${bracketData.open};${bracketData.close}`
));

startOffset = foundBracketEndOffset;
Expand All @@ -263,8 +263,8 @@ class TokenScanner {
// there is some remaining none-bracket text in this token
this._nextBuff.push(new Token(
new Range(lineNumber, startOffset + 1, lineNumber, endOffset + 1),
tokenType,
TokenTreeBracket.None
TokenTreeBracket.None,
null
));
}

Expand Down Expand Up @@ -353,7 +353,7 @@ class TokenTreeBuilder {
accepted: boolean;

accepted = this._accept(token => {
bracketType = token.type;
bracketType = token.bracketType;
return token.bracket === TokenTreeBracket.Open;
});
if (!accepted) {
Expand All @@ -366,7 +366,7 @@ class TokenTreeBuilder {
// inside brackets
}

if (!this._accept(token => token.bracket === TokenTreeBracket.Close && token.type === bracketType)) {
if (!this._accept(token => token.bracket === TokenTreeBracket.Close && token.bracketType === bracketType)) {
// missing closing bracket -> return just a node list
var nodelist = new NodeList();
nodelist.append(bracket.open);
Expand Down

0 comments on commit b70ce79

Please sign in to comment.