[vscode] semantic highlighting support

packages/editor/src/browser/editor-preferences.ts

@@ -109,7 +109,7 @@ const codeEditorPreferenceProperties = {
     },
     'editor.semanticHighlighting.enabled': {
         'type': 'boolean',
-        'default': false,
+        'default': true,
         'description': 'Controls whether the semanticHighlighting is shown for the languages that support it.'
     },
     'editor.stablePeek': {

packages/monaco/src/browser/monaco-editor-provider.ts

@@ -95,6 +95,32 @@ export class MonacoEditorProvider {
         const staticServices = monaco.services.StaticServices;
         const init = staticServices.init.bind(monaco.services.StaticServices);
 
+        const themeService = staticServices.standaloneThemeService.get();
+        const originalGetTheme: (typeof themeService)['getTheme'] = themeService.getTheme.bind(themeService);
+        const patchedGetTokenStyleMetadataFlag = '__patched_getTokenStyleMetadata';
+        // based on https://github.com/microsoft/vscode/commit/4731a227e377da8cb14ed5697dd1ba8faea40538
+        // TODO remove after migrating to monaco 0.21
+        themeService.getTheme = () => {
+            const theme = originalGetTheme();
+            if (!(patchedGetTokenStyleMetadataFlag in theme)) {
+                Object.defineProperty(theme, patchedGetTokenStyleMetadataFlag, { enumerable: false, configurable: false, writable: false, value: true });
+                theme.getTokenStyleMetadata = (type, modifiers) => {
+                    // use theme rules match
+                    const style = theme.tokenTheme._match([type].concat(modifiers).join('.'));
+                    const metadata = style.metadata;
+                    const foreground = monaco.modes.TokenMetadata.getForeground(metadata);
+                    const fontStyle = monaco.modes.TokenMetadata.getFontStyle(metadata);
+                    return {
+                        foreground: foreground,
+                        italic: Boolean(fontStyle & monaco.modes.FontStyle.Italic),
+                        bold: Boolean(fontStyle & monaco.modes.FontStyle.Bold),
+                        underline: Boolean(fontStyle & monaco.modes.FontStyle.Underline)
+                    };
+                };
+            }
+            return theme;
+        };
+
         monaco.services.StaticServices.init = o => {
             const result = init(o);
             result[0].set(monaco.services.ICodeEditorService, codeEditorService);

packages/monaco/src/typings/monaco/index.d.ts

@@ -762,6 +762,14 @@ declare module monaco.services {
         get(overrides?: monaco.editor.IEditorOverrideServices): T;
     }
 
+    // https://github.com/microsoft/vscode/blob/0eb3a02ca2bcfab5faa3dc6e52d7c079efafcab0/src/vs/platform/theme/common/themeService.ts#L78
+    export interface ITokenStyle {
+        readonly foreground?: number;
+        readonly bold?: boolean;
+        readonly underline?: boolean;
+        readonly italic?: boolean;
+    }
+
     // https://github.com/theia-ide/vscode/blob/standalone/0.20.x/src/vs/editor/standalone/common/standaloneThemeService.ts#L28
     export interface IStandaloneThemeService extends monaco.theme.IThemeService {
         // https://github.com/theia-ide/vscode/blob/standalone/0.20.x/src/vs/editor/standalone/browser/standaloneThemeServiceImpl.ts#L178
@@ -779,11 +787,14 @@ declare module monaco.services {
 
         // https://github.com/theia-ide/vscode/blob/standalone/0.20.x/src/vs/platform/theme/common/themeService.ts#L98
         getColor(color: string, useDefault?: boolean): monaco.color.Color | undefined;
+
+        getTokenStyleMetadata(type: string, modifiers: string[], modelLanguage: string): ITokenStyle | undefined;
     }
 
     // https://github.com/theia-ide/vscode/blob/standalone/0.20.x/src/vs/editor/common/modes/supports/tokenization.ts#L188
     export interface TokenTheme {
         match(languageId: LanguageId, scope: string): number;
+        _match(token: string): any;
         getColorMap(): monaco.color.Color[];
     }
 
@@ -1301,6 +1312,15 @@ declare module monaco.modes {
     }
     export const TokenizationRegistry: TokenizationRegistry;
 
+    // https://github.com/microsoft/vscode/blob/0eb3a02ca2bcfab5faa3dc6e52d7c079efafcab0/src/vs/editor/common/modes.ts#L66-L76
+    export const enum FontStyle {
+        NotSet = -1,
+        None = 0,
+        Italic = 1,
+        Bold = 2,
+        Underline = 4
+    }
+
     // https://github.com/theia-ide/vscode/blob/standalone/0.20.x/src/vs/editor/common/modes.ts#L148
     export class TokenMetadata {
 

packages/plugin-ext/src/common/plugin-api-rpc.ts

@@ -1279,6 +1279,9 @@ export interface LanguagesExt {
     $provideColorPresentations(handle: number, resource: UriComponents, colorInfo: RawColorInfo, token: CancellationToken): PromiseLike<ColorPresentation[]>;
     $provideRenameEdits(handle: number, resource: UriComponents, position: Position, newName: string, token: CancellationToken): PromiseLike<WorkspaceEditDto | undefined>;
     $resolveRenameLocation(handle: number, resource: UriComponents, position: Position, token: CancellationToken): PromiseLike<RenameLocation | undefined>;
+    $provideDocumentSemanticTokens(handle: number, resource: UriComponents, previousResultId: number, token: CancellationToken): Promise<BinaryBuffer | null>;
+    $releaseDocumentSemanticTokens(handle: number, semanticColoringResultId: number): void;
+    $provideDocumentRangeSemanticTokens(handle: number, resource: UriComponents, range: Range, token: CancellationToken): Promise<BinaryBuffer | null>;
     $provideRootDefinition(handle: number, resource: UriComponents, location: Position, token: CancellationToken): Promise<CallHierarchyDefinition | undefined>;
     $provideCallers(handle: number, definition: CallHierarchyDefinition, token: CancellationToken): Promise<CallHierarchyReference[] | undefined>;
 }
@@ -1323,6 +1326,10 @@ export interface LanguagesMain {
     $registerSelectionRangeProvider(handle: number, pluginInfo: PluginInfo, selector: SerializedDocumentFilter[]): void;
     $registerDocumentColorProvider(handle: number, pluginInfo: PluginInfo, selector: SerializedDocumentFilter[]): void;
     $registerRenameProvider(handle: number, pluginInfo: PluginInfo, selector: SerializedDocumentFilter[], supportsResolveInitialValues: boolean): void;
+    $registerDocumentSemanticTokensProvider(handle: number, pluginInfo: PluginInfo, selector: SerializedDocumentFilter[],
+        legend: theia.SemanticTokensLegend, eventHandle: number | undefined): void;
+    $emitDocumentSemanticTokensEvent(eventHandle: number): void;
+    $registerDocumentRangeSemanticTokensProvider(handle: number, pluginInfo: PluginInfo, selector: SerializedDocumentFilter[], legend: theia.SemanticTokensLegend): void;
     $registerCallHierarchyProvider(handle: number, selector: SerializedDocumentFilter[]): void;
 }
 
@@ -1549,7 +1556,7 @@ export interface AuthenticationMain {
     $getProviderIds(): Promise<string[]>;
     $updateSessions(providerId: string, event: AuthenticationSessionsChangeEvent): void;
     $getSession(providerId: string, scopes: string[], extensionId: string, extensionName: string,
-                options: { createIfNone?: boolean, clearSessionPreference?: boolean }): Promise<theia.AuthenticationSession | undefined>;
+        options: { createIfNone?: boolean, clearSessionPreference?: boolean }): Promise<theia.AuthenticationSession | undefined>;
     $logout(providerId: string, sessionId: string): Promise<void>;
 }
 

packages/plugin-ext/src/common/semantic-tokens-dto.ts

@@ -0,0 +1,182 @@
+/********************************************************************************
+ * Copyright (C) 2020 TypeFox and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+// copied and modified from https://github.com/microsoft/vscode/blob/0eb3a02ca2bcfab5faa3dc6e52d7c079efafcab0/src/vs/workbench/api/common/shared/semanticTokensDto.ts
+
+import { BinaryBuffer } from '@theia/core/lib/common/buffer';
+
+let _isLittleEndian = true;
+let _isLittleEndianComputed = false;
+function isLittleEndian(): boolean {
+    if (!_isLittleEndianComputed) {
+        _isLittleEndianComputed = true;
+        const test = new Uint8Array(2);
+        test[0] = 1;
+        test[1] = 2;
+        const view = new Uint16Array(test.buffer);
+        _isLittleEndian = (view[0] === (2 << 8) + 1);
+    }
+    return _isLittleEndian;
+}
+
+export interface IFullSemanticTokensDto {
+    id: number;
+    type: 'full';
+    data: Uint32Array;
+}
+
+export interface IDeltaSemanticTokensDto {
+    id: number;
+    type: 'delta';
+    deltas: { start: number; deleteCount: number; data?: Uint32Array; }[];
+}
+
+export type ISemanticTokensDto = IFullSemanticTokensDto | IDeltaSemanticTokensDto;
+
+const enum EncodedSemanticTokensType {
+    Full = 1,
+    Delta = 2
+}
+
+function reverseEndianness(arr: Uint8Array): void {
+    for (let i = 0, len = arr.length; i < len; i += 4) {
+        // flip bytes 0<->3 and 1<->2
+        const b0 = arr[i + 0];
+        const b1 = arr[i + 1];
+        const b2 = arr[i + 2];
+        const b3 = arr[i + 3];
+        arr[i + 0] = b3;
+        arr[i + 1] = b2;
+        arr[i + 2] = b1;
+        arr[i + 3] = b0;
+    }
+}
+
+function toLittleEndianBuffer(arr: Uint32Array): BinaryBuffer {
+    const uint8Arr = new Uint8Array(arr.buffer, arr.byteOffset, arr.length * 4);
+    if (!isLittleEndian()) {
+        // the byte order must be changed
+        reverseEndianness(uint8Arr);
+    }
+    return BinaryBuffer.wrap(uint8Arr);
+}
+
+function fromLittleEndianBuffer(buff: BinaryBuffer): Uint32Array {
+    const uint8Arr = buff.buffer;
+    if (!isLittleEndian()) {
+        // the byte order must be changed
+        reverseEndianness(uint8Arr);
+    }
+    if (uint8Arr.byteOffset % 4 === 0) {
+        return new Uint32Array(uint8Arr.buffer, uint8Arr.byteOffset, uint8Arr.length / 4);
+    } else {
+        // unaligned memory access doesn't work on all platforms
+        const data = new Uint8Array(uint8Arr.byteLength);
+        data.set(uint8Arr);
+        return new Uint32Array(data.buffer, data.byteOffset, data.length / 4);
+    }
+}
+
+export function encodeSemanticTokensDto(semanticTokens: ISemanticTokensDto): BinaryBuffer {
+    const dest = new Uint32Array(encodeSemanticTokensDtoSize(semanticTokens));
+    let offset = 0;
+    dest[offset++] = semanticTokens.id;
+    if (semanticTokens.type === 'full') {
+        dest[offset++] = EncodedSemanticTokensType.Full;
+        dest[offset++] = semanticTokens.data.length;
+        dest.set(semanticTokens.data, offset); offset += semanticTokens.data.length;
+    } else {
+        dest[offset++] = EncodedSemanticTokensType.Delta;
+        dest[offset++] = semanticTokens.deltas.length;
+        for (const delta of semanticTokens.deltas) {
+            dest[offset++] = delta.start;
+            dest[offset++] = delta.deleteCount;
+            if (delta.data) {
+                dest[offset++] = delta.data.length;
+                dest.set(delta.data, offset); offset += delta.data.length;
+            } else {
+                dest[offset++] = 0;
+            }
+        }
+    }
+    return toLittleEndianBuffer(dest);
+}
+
+function encodeSemanticTokensDtoSize(semanticTokens: ISemanticTokensDto): number {
+    let result = 0;
+    result += (
+        + 1 // id
+        + 1 // type
+    );
+    if (semanticTokens.type === 'full') {
+        result += (
+            + 1 // data length
+            + semanticTokens.data.length
+        );
+    } else {
+        result += (
+            + 1 // delta count
+        );
+        result += (
+            + 1 // start
+            + 1 // deleteCount
+            + 1 // data length
+        ) * semanticTokens.deltas.length;
+        for (const delta of semanticTokens.deltas) {
+            if (delta.data) {
+                result += delta.data.length;
+            }
+        }
+    }
+    return result;
+}
+
+export function decodeSemanticTokensDto(_buff: BinaryBuffer): ISemanticTokensDto {
+    const src = fromLittleEndianBuffer(_buff);
+    let offset = 0;
+    const id = src[offset++];
+    const type: EncodedSemanticTokensType = src[offset++];
+    if (type === EncodedSemanticTokensType.Full) {
+        const length = src[offset++];
+        const data = src.subarray(offset, offset + length); offset += length;
+        return {
+            id: id,
+            type: 'full',
+            data: data
+        };
+    }
+    const deltaCount = src[offset++];
+    const deltas: { start: number; deleteCount: number; data?: Uint32Array; }[] = [];
+    for (let i = 0; i < deltaCount; i++) {
+        const start = src[offset++];
+        const deleteCount = src[offset++];
+        const length = src[offset++];
+        let data: Uint32Array | undefined;
+        if (length > 0) {
+            data = src.subarray(offset, offset + length); offset += length;
+        }
+        deltas[i] = { start, deleteCount, data };
+    }
+    return {
+        id: id,
+        type: 'delta',
+        deltas: deltas
+    };
+}