Skip to content

Commit

Permalink
GH-2574: Generalized the semantic highlighting API and feature.
Browse files Browse the repository at this point in the history
Also fixed a decoding bug.

Closes: #2574.

Signed-off-by: Akos Kitta <[email protected]>
  • Loading branch information
Akos Kitta committed Aug 14, 2018
1 parent d43c0ac commit c50113d
Show file tree
Hide file tree
Showing 7 changed files with 198 additions and 88 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,16 @@ describe('semantic-highlighting-service', () => {
expect(actual).to.be.deep.equal(expected);
});

it('should fill with zeros when right shift for the decode phase', function () {
this.timeout(10_000);
const input: number[] = [];
for (let i = 0; i < 65_536; i++) {
input.push(...[i, i, i]);
}
const expected = SemanticHighlightingService.Token.fromArray(input);
const encoded = SemanticHighlightingService.encode(expected);
const actual = SemanticHighlightingService.decode(encoded);
expect(actual).to.be.deep.equal(expected);
});

});
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ import { Position, Range } from 'vscode-languageserver-types';
import URI from '@theia/core/lib/common/uri';
import { Disposable } from '@theia/core/lib/common/disposable';
import { ILogger } from '@theia/core/lib/common/logger';
import { ILanguageClient } from '@theia/languages/lib/browser/language-client-services';
import { SemanticHighlightFeature } from '@theia/languages/lib/browser/semantic-highlighting/semantic-highlighting-feature';
import { SemanticHighlightingParams } from '@theia/languages/lib/browser/semantic-highlighting/semantic-highlighting-protocol';

/**
* Service for registering and managing semantic highlighting decorations in the code editors for multiple languages.
Expand Down Expand Up @@ -164,7 +167,7 @@ export namespace SemanticHighlightingService {
for (let i = 0; i < buffer.byteLength / Uint32Array.BYTES_PER_ELEMENT; i = i + 2) {
const character = dataView.getUint32(i * Uint32Array.BYTES_PER_ELEMENT);
const lengthAndScope = dataView.getUint32((i + 1) * Uint32Array.BYTES_PER_ELEMENT);
const length = lengthAndScope >> LENGTH_SHIFT;
const length = lengthAndScope >>> LENGTH_SHIFT;
const scope = lengthAndScope & SCOPE_MASK;
result.push({
character,
Expand Down Expand Up @@ -198,6 +201,39 @@ export namespace SemanticHighlightingService {
return base64Encode(buffer);
}

/**
* Creates a new text document feature to handle the semantic highlighting capabilities of the protocol.
* When the feature gets initialized, it delegates to the semantic highlighting service and registers the TextMate scopes received as part of the server capabilities.
* Plus, each time when a notification is received by the client, the semantic highlighting service will be used as the consumer and the highlighting information
* will be used to decorate the text editor.
*/
export function createNewFeature(service: SemanticHighlightingService, client: ILanguageClient & Readonly<{ languageId: string }>): SemanticHighlightFeature {
const { languageId } = client;
const initializeCallback = (id: string, scopes: string[][]): Disposable => service.register(id, scopes);
const consumer = (params: SemanticHighlightingParams): void => {
const toRanges: (tuple: [number, string | undefined]) => SemanticHighlightingRange[] = tuple => {
const [line, tokens] = tuple;
if (!tokens) {
return [
{
start: Position.create(line, 0),
end: Position.create(line, 0),
}
];
}
return SemanticHighlightingService.decode(tokens).map(token => ({
start: Position.create(line, token.character),
end: Position.create(line, token.character + token.length),
scope: token.scope
}));
};
const ranges = params.lines.map(line => [line.line, line.tokens]).map(toRanges).reduce((acc, current) => acc.concat(current), []);
const uri = new URI(params.textDocument.uri);
service.decorate(languageId, uri, ranges);
};
return new SemanticHighlightFeature(client, initializeCallback, consumer);
}

}

export interface SemanticHighlightingRange extends Range {
Expand Down
4 changes: 1 addition & 3 deletions packages/java/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,10 @@
"@theia/monaco": "^0.3.13",
"@types/glob": "^5.0.30",
"@types/tar": "4.0.0",
"@types/uuid": "^3.4.3",
"glob": "^7.1.2",
"mkdirp": "^0.5.0",
"sha1": "^1.1.1",
"tar": "^4.0.0",
"uuid": "^3.2.1"
"tar": "^4.0.0"
},
"devDependencies": {
"@theia/ext-scripts": "^0.3.13"
Expand Down
87 changes: 4 additions & 83 deletions packages/java/src/browser/java-client-contribution.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,33 +15,23 @@
********************************************************************************/

import { injectable, inject } from 'inversify';
import { v4 } from 'uuid';
import URI from '@theia/core/lib/common/uri';
import { DisposableCollection, CommandService } from '@theia/core/lib/common/';
import { CommandService } from '@theia/core/lib/common/';
import { StatusBar, StatusBarEntry, StatusBarAlignment } from '@theia/core/lib/browser';
import { SemanticHighlightingService, SemanticHighlightingRange, Position } from '@theia/editor/lib/browser/semantic-highlight/semantic-highlighting-service';
import { SemanticHighlightingService } from '@theia/editor/lib/browser/semantic-highlight/semantic-highlighting-service';
import {
Window,
ILanguageClient,
BaseLanguageClientContribution,
Workspace, Languages,
LanguageClientFactory,
LanguageClientOptions,
TextDocumentFeature,
TextDocumentRegistrationOptions,
ClientCapabilities,
ServerCapabilities,
Disposable,
DocumentSelector
LanguageClientOptions
} from '@theia/languages/lib/browser';
import { JAVA_LANGUAGE_ID, JAVA_LANGUAGE_NAME } from '../common';
import {
ActionableNotification,
ActionableMessage,
StatusReport,
StatusNotification,
SemanticHighlight,
SemanticHighlightingParams
} from './java-protocol';

@injectable()
Expand Down Expand Up @@ -80,7 +70,7 @@ export class JavaClientContribution extends BaseLanguageClientContribution {

createLanguageClient(): ILanguageClient {
const client: ILanguageClient & Readonly<{ languageId: string }> = Object.assign(super.createLanguageClient(), { languageId: this.id });
client.registerFeature(new SemanticHighlightFeature(client, this.semanticHighlightingService));
client.registerFeature(SemanticHighlightingService.createNewFeature(this.semanticHighlightingService, client));
return client;
}

Expand Down Expand Up @@ -123,72 +113,3 @@ export class JavaClientContribution extends BaseLanguageClientContribution {
}

}

// TODO: This will be part of the protocol.
export class SemanticHighlightFeature extends TextDocumentFeature<TextDocumentRegistrationOptions> {

protected readonly languageId: string;
protected readonly toDispose: DisposableCollection;

constructor(client: ILanguageClient & Readonly<{ languageId: string }>, protected readonly semanticHighlightingService: SemanticHighlightingService) {
super(client, SemanticHighlight.type);
this.languageId = client.languageId;
this.toDispose = new DisposableCollection();
}

fillClientCapabilities(capabilities: ClientCapabilities): void {
if (!capabilities.textDocument) {
capabilities.textDocument = {};
}
// tslint:disable-next-line:no-any
(capabilities.textDocument as any).semanticHighlightingCapabilities = {
semanticHighlighting: true
};
}

initialize(capabilities: ServerCapabilities, documentSelector: DocumentSelector): void {
if (!documentSelector) {
return;
}
const capabilitiesExt: ServerCapabilities & { semanticHighlighting?: { scopes: string[][] | undefined } } = capabilities;
if (capabilitiesExt.semanticHighlighting) {
const { scopes } = capabilitiesExt.semanticHighlighting;
if (scopes && scopes.length > 0) {
this.toDispose.push(this.semanticHighlightingService.register(this.languageId, scopes));
const id = v4();
this.register(this.messages, {
id,
registerOptions: Object.assign({}, { documentSelector: documentSelector }, capabilitiesExt.semanticHighlighting)
});
}
}
}

protected registerLanguageProvider(): Disposable {
this._client.onNotification(SemanticHighlight.type, this.applySemanticHighlighting.bind(this));
return Disposable.create(() => this.toDispose.dispose());
}

protected applySemanticHighlighting(params: SemanticHighlightingParams): void {
const toRanges: (tuple: [number, string | undefined]) => SemanticHighlightingRange[] = tuple => {
const [line, tokens] = tuple;
if (!tokens) {
return [
{
start: Position.create(line, 0),
end: Position.create(line, 0),
}
];
}
return SemanticHighlightingService.decode(tokens).map(token => ({
start: Position.create(line, token.character),
end: Position.create(line, token.character + token.length),
scope: token.scope
}));
};
const ranges = params.lines.map(line => [line.line, line.tokens]).map(toRanges).reduce((acc, current) => acc.concat(current), []);
const uri = new URI(params.textDocument.uri);
this.semanticHighlightingService.decorate(this.languageId, uri, ranges);
}

}
4 changes: 3 additions & 1 deletion packages/languages/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
"@theia/output": "^0.3.13",
"@theia/process": "^0.3.13",
"@theia/workspace": "^0.3.13",
"monaco-languageclient": "next"
"@types/uuid": "^3.4.3",
"monaco-languageclient": "next",
"uuid": "^3.2.1"
},
"publishConfig": {
"access": "public"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
/********************************************************************************
* Copyright (C) 2018 TypeFox and others.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v. 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* This Source Code may also be made available under the following Secondary
* Licenses when the conditions for such availability set forth in the Eclipse
* Public License v. 2.0 are satisfied: GNU General Public License, version 2
* with the GNU Classpath Exception which is available at
* https://www.gnu.org/software/classpath/license.html.
*
* SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
********************************************************************************/

import { v4 } from 'uuid';
import { DisposableCollection } from '@theia/core/lib/common/';
import {
ILanguageClient,
TextDocumentFeature,
TextDocumentRegistrationOptions,
ClientCapabilities,
ServerCapabilities,
Disposable,
DocumentSelector
} from '../';
import { SemanticHighlight, SemanticHighlightingParams } from './semantic-highlighting-protocol';

// NOTE: This module can be removed, or at least can be simplified once the semantic highlighting will become the part of the LSP.
// https://github.com/Microsoft/vscode-languageserver-node/issues/368

export class SemanticHighlightFeature extends TextDocumentFeature<TextDocumentRegistrationOptions> {

protected readonly languageId: string;
protected readonly toDispose: DisposableCollection;

constructor(
client: ILanguageClient & Readonly<{ languageId: string }>,
protected readonly initializeCallback: SemanticHighlightFeature.SemanticHighlightFeatureInitializeCallback,
protected readonly consumer: SemanticHighlightFeature.SemanticHighlightingParamsConsumer) {

super(client, SemanticHighlight.type);
this.languageId = client.languageId;
this.toDispose = new DisposableCollection();
}

fillClientCapabilities(capabilities: ClientCapabilities): void {
if (!capabilities.textDocument) {
capabilities.textDocument = {};
}
// tslint:disable-next-line:no-any
(capabilities.textDocument as any).semanticHighlightingCapabilities = {
semanticHighlighting: true
};
}

initialize(capabilities: ServerCapabilities, documentSelector: DocumentSelector): void {
if (!documentSelector) {
return;
}
const capabilitiesExt: ServerCapabilities & { semanticHighlighting?: { scopes: string[][] | undefined } } = capabilities;
if (capabilitiesExt.semanticHighlighting) {
const { scopes } = capabilitiesExt.semanticHighlighting;
if (scopes && scopes.length > 0) {
this.toDispose.push(this.initializeCallback(this.languageId, scopes));
const id = v4();
this.register(this.messages, {
id,
registerOptions: Object.assign({}, { documentSelector: documentSelector }, capabilitiesExt.semanticHighlighting)
});
}
}
}

protected registerLanguageProvider(): Disposable {
this._client.onNotification(SemanticHighlight.type, this.consumeSemanticHighlighting.bind(this));
return Disposable.create(() => this.toDispose.dispose());
}

protected consumeSemanticHighlighting(params: SemanticHighlightingParams): void {
this.consumer(params);
}

}

export namespace SemanticHighlightFeature {

export interface SemanticHighlightingParamsConsumer {

/**
* Consumes a semantic highlighting notification, received from the language server.
*/
(params: SemanticHighlightingParams): void
}

export interface SemanticHighlightFeatureInitializeCallback {

/**
* Invoked when the connection between the client and the server has been established and the server sends back
* a "lookup table" of TextMate scopes.
*/
(languageId: string, scopes: string[][]): Disposable
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/********************************************************************************
* Copyright (C) 2018 TypeFox and others.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License v. 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* This Source Code may also be made available under the following Secondary
* Licenses when the conditions for such availability set forth in the Eclipse
* Public License v. 2.0 are satisfied: GNU General Public License, version 2
* with the GNU Classpath Exception which is available at
* https://www.gnu.org/software/classpath/license.html.
*
* SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
********************************************************************************/

import { NotificationType } from 'vscode-jsonrpc';
import { VersionedTextDocumentIdentifier } from '..';

// NOTE: This module can be removed, once the semantic highlighting will become the part of the LSP.
// https://github.com/Microsoft/vscode-languageserver-node/issues/368

export interface SemanticHighlightingParams {
readonly textDocument: VersionedTextDocumentIdentifier;
readonly lines: SemanticHighlightingInformation[];
}

export interface SemanticHighlightingInformation {
readonly line: number;
readonly tokens?: string;
}

export namespace SemanticHighlight {
export const type = new NotificationType<SemanticHighlightingParams, void>('textDocument/semanticHighlighting');
}

0 comments on commit c50113d

Please sign in to comment.