Add missing operators in tokenizer, handle end-of-comment completion …

…better (#1129)

* Basic tokenizer

* Fixed property names

* Tests, round I

* Tests, round II

* tokenizer test

* Remove temorary change

* Fix merge issue

* Merge conflict

* Merge conflict

* Completion test

* Fix last line

* Fix javascript math

* Make test await for results

* Add license headers

* Rename definitions to types

* License headers

* Fix typo in completion details (typo)

* Fix hover test

* Russian translations

* Update to better translation

* Fix typo

*  #70 How to get all parameter info when filling in a function param list

* Fix #70 How to get all parameter info when filling in a function param list

* Clean up

* Clean imports

* CR feedback

* Trim whitespace for test stability

* More tests

* Better handle no-parameters documentation

* Better handle ellipsis and Python3

* #385 Auto-Indentation doesn't work after comment

* #141 Auto indentation broken when return keyword involved

* Undo changes

* #627 Docstrings for builtin methods are not parsed correctly

* reStructuredText converter

* Fix: period is not an operator

* Minor fixes

* Restructure

* Tests

* Tests

* Code heuristics

* Baselines

* HTML handling

* Lists

* State machine

* Baselines

* Squash

* no message

* Whitespace difference

* Update Jedi to 0.11.1

* Enable Travis

* Test fixes

* Undo change

* Jedi 0.11 with parser

* Undo changes

* Undo changes

* Test fixes

* More tests

* Tests

* Fix pylint search

* Handle quote escapes in strings

* Escapes in strings

* CR feedback

* Discover pylintrc better + tests

* Fix .pyenv/versions search

* Fix multiple linters output

* Better handle markdown underscore

* Test

* Fix 916: PyLint checks wrong files

* Test stability

* Try increase timeout

* Make sure linting is enabled in tests

* Try another way of waiting

* Simplify

* Fix clear diags on close tests

* Try writing settings directly

* Increase timeout

* Measure test time

* Measure time

* Simplify

* Set timeout

* Better venv detection

* Add test

* More reliable check

* Fix pylint switch key

* Remove incorrect flag

* Disable print

* Require pylint 1.8 on CI

* Fix working directory for standalone files

* Use an 'elif'

* Separate file for pylint root config

* Remove double event listening

* Remove test

* Revert "Remove test"

This reverts commit e240c3f.

* Revert "Remove double event listening"

This reverts commit af573be.

* Explicitly disable linter

* New buttons

* PR feedback

* Revert "PR feedback"

This reverts commit d0c7ab5.

* PR feedback

* Fix end of comment completion and operator handling

* Simplify

* Test typo fix

.gitignore

@@ -15,3 +15,4 @@ coverage/
 .venv
 pythonFiles/experimental/ptvsd/**
 debug_coverage*/**
+analysis/**

src/client/extension.ts

@@ -6,10 +6,12 @@ if ((Reflect as any).metadata === undefined) {
     require('reflect-metadata');
 }
 import { Container } from 'inversify';
-import * as vscode from 'vscode';
-import { Disposable, Memento, OutputChannel, window } from 'vscode';
+import {
+    debug, Disposable, DocumentFilter, ExtensionContext,
+    extensions, IndentAction, languages, Memento,
+    OutputChannel, window
+} from 'vscode';
 import { PythonSettings } from './common/configSettings';
-import * as settings from './common/configSettings';
 import { STANDARD_OUTPUT_CHANNEL } from './common/constants';
 import { FeatureDeprecationManager } from './common/featureDeprecationManager';
 import { createDeferred } from './common/helpers';
@@ -61,12 +63,12 @@ import * as tests from './unittests/main';
 import { registerTypes as unitTestsRegisterTypes } from './unittests/serviceRegistry';
 import { WorkspaceSymbols } from './workspaceSymbols/main';
 
-const PYTHON: vscode.DocumentFilter = { language: 'python' };
+const PYTHON: DocumentFilter = { language: 'python' };
 const activationDeferred = createDeferred<void>();
 export const activated = activationDeferred.promise;
 
 // tslint:disable-next-line:max-func-body-length
-export async function activate(context: vscode.ExtensionContext) {
+export async function activate(context: ExtensionContext) {
     const cont = new Container();
     const serviceManager = new ServiceManager(cont);
     const serviceContainer = new ServiceContainer(cont);
@@ -95,7 +97,7 @@ export async function activate(context: vscode.ExtensionContext) {
     serviceManager.get<ICodeExecutionManager>(ICodeExecutionManager).registerCommands();
 
     const persistentStateFactory = serviceManager.get<IPersistentStateFactory>(IPersistentStateFactory);
-    const pythonSettings = settings.PythonSettings.getInstance();
+    const pythonSettings = PythonSettings.getInstance();
     // tslint:disable-next-line:no-floating-promises
     sendStartupTelemetry(activated, serviceContainer);
 
@@ -125,60 +127,60 @@ export async function activate(context: vscode.ExtensionContext) {
 
     // Enable indentAction
     // tslint:disable-next-line:no-non-null-assertion
-    vscode.languages.setLanguageConfiguration(PYTHON.language!, {
+    languages.setLanguageConfiguration(PYTHON.language!, {
         onEnterRules: [
             {
                 beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async)\b.*/,
-                action: { indentAction: vscode.IndentAction.Indent }
+                action: { indentAction: IndentAction.Indent }
             },
             {
                 beforeText: /^\s*#.*/,
                 afterText: /.+$/,
-                action: { indentAction: vscode.IndentAction.None, appendText: '# ' }
+                action: { indentAction: IndentAction.None, appendText: '# ' }
             },
             {
                 beforeText: /^\s+(continue|break|return)\b.*/,
                 afterText: /\s+$/,
-                action: { indentAction: vscode.IndentAction.Outdent }
+                action: { indentAction: IndentAction.Outdent }
             }
         ]
     });
 
     context.subscriptions.push(jediFactory);
-    context.subscriptions.push(vscode.languages.registerRenameProvider(PYTHON, new PythonRenameProvider(serviceContainer)));
+    context.subscriptions.push(languages.registerRenameProvider(PYTHON, new PythonRenameProvider(serviceContainer)));
     const definitionProvider = new PythonDefinitionProvider(jediFactory);
-    context.subscriptions.push(vscode.languages.registerDefinitionProvider(PYTHON, definitionProvider));
-    context.subscriptions.push(vscode.languages.registerHoverProvider(PYTHON, new PythonHoverProvider(jediFactory)));
-    context.subscriptions.push(vscode.languages.registerReferenceProvider(PYTHON, new PythonReferenceProvider(jediFactory)));
-    context.subscriptions.push(vscode.languages.registerCompletionItemProvider(PYTHON, new PythonCompletionItemProvider(jediFactory, serviceContainer), '.'));
-    context.subscriptions.push(vscode.languages.registerCodeLensProvider(PYTHON, serviceContainer.get<IShebangCodeLensProvider>(IShebangCodeLensProvider)));
+    context.subscriptions.push(languages.registerDefinitionProvider(PYTHON, definitionProvider));
+    context.subscriptions.push(languages.registerHoverProvider(PYTHON, new PythonHoverProvider(jediFactory)));
+    context.subscriptions.push(languages.registerReferenceProvider(PYTHON, new PythonReferenceProvider(jediFactory)));
+    context.subscriptions.push(languages.registerCompletionItemProvider(PYTHON, new PythonCompletionItemProvider(jediFactory, serviceContainer), '.'));
+    context.subscriptions.push(languages.registerCodeLensProvider(PYTHON, serviceContainer.get<IShebangCodeLensProvider>(IShebangCodeLensProvider)));
 
     const symbolProvider = new PythonSymbolProvider(jediFactory);
-    context.subscriptions.push(vscode.languages.registerDocumentSymbolProvider(PYTHON, symbolProvider));
+    context.subscriptions.push(languages.registerDocumentSymbolProvider(PYTHON, symbolProvider));
     if (pythonSettings.devOptions.indexOf('DISABLE_SIGNATURE') === -1) {
-        context.subscriptions.push(vscode.languages.registerSignatureHelpProvider(PYTHON, new PythonSignatureProvider(jediFactory), '(', ','));
+        context.subscriptions.push(languages.registerSignatureHelpProvider(PYTHON, new PythonSignatureProvider(jediFactory), '(', ','));
     }
     if (pythonSettings.formatting.provider !== 'none') {
         const formatProvider = new PythonFormattingEditProvider(context, serviceContainer);
-        context.subscriptions.push(vscode.languages.registerDocumentFormattingEditProvider(PYTHON, formatProvider));
-        context.subscriptions.push(vscode.languages.registerDocumentRangeFormattingEditProvider(PYTHON, formatProvider));
+        context.subscriptions.push(languages.registerDocumentFormattingEditProvider(PYTHON, formatProvider));
+        context.subscriptions.push(languages.registerDocumentRangeFormattingEditProvider(PYTHON, formatProvider));
     }
 
     const linterProvider = new LinterProvider(context, serviceContainer);
     context.subscriptions.push(linterProvider);
 
-    const jupyterExtension = vscode.extensions.getExtension('donjayamanne.jupyter');
+    const jupyterExtension = extensions.getExtension('donjayamanne.jupyter');
     const lintingEngine = serviceContainer.get<ILintingEngine>(ILintingEngine);
     lintingEngine.linkJupiterExtension(jupyterExtension).ignoreErrors();
 
     tests.activate(context, unitTestOutChannel, symbolProvider, serviceContainer);
 
     context.subscriptions.push(new WorkspaceSymbols(serviceContainer));
-    context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':'));
-    context.subscriptions.push(vscode.languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n'));
+    context.subscriptions.push(languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':'));
+    context.subscriptions.push(languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n'));
 
     serviceContainer.getAll<BaseConfigurationProvider>(IDebugConfigurationProvider).forEach(debugConfig => {
-        context.subscriptions.push(vscode.debug.registerDebugConfigurationProvider(debugConfig.debugType, debugConfig));
+        context.subscriptions.push(debug.registerDebugConfigurationProvider(debugConfig.debugType, debugConfig));
     });
     activationDeferred.resolve();
 

src/client/language/tokenizer.ts

@@ -34,10 +34,10 @@ export class Tokenizer implements ITokenizer {
     //     'not', 'or', 'pass', 'print', 'raise', 'return', 'True', 'try',
     //     'while', 'with', 'yield'
     // ];
-    private cs: ICharacterStream;
+    private cs: ICharacterStream = new CharacterStream('');
     private tokens: IToken[] = [];
     private floatRegex = /[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?/;
-    private mode: TokenizerMode;
+    private mode = TokenizerMode.Full;
 
     constructor() {
         //this.floatRegex.compile();
@@ -287,15 +287,15 @@ export class Tokenizer implements ITokenizer {
                 } else if (nextChar === Char.Less) {
                     length = this.cs.lookAhead(2) === Char.Equal ? 3 : 2;
                 } else {
-                    length = 1;
+                    length = nextChar === Char.Equal ? 2 : 1;
                 }
                 break;
 
             case Char.Greater:
                 if (nextChar === Char.Greater) {
                     length = this.cs.lookAhead(2) === Char.Equal ? 3 : 2;
                 } else {
-                    length = 1;
+                    length = nextChar === Char.Equal ? 2 : 1;
                 }
                 break;
 

src/client/providers/completionSource.ts

@@ -61,7 +61,7 @@ export class CompletionSource {
         const sourceText = `${document.getText(leadingRange)}${itemString}`;
         const range = new vscode.Range(leadingRange.end, leadingRange.end.translate(0, itemString.length));
 
-        return await this.itemInfoSource.getItemInfoFromText(document.uri, document.fileName, range, sourceText, token);
+        return this.itemInfoSource.getItemInfoFromText(document.uri, document.fileName, range, sourceText, token);
     }
 
     private async getCompletionResult(document: vscode.TextDocument, position: vscode.Position, token: vscode.CancellationToken)
@@ -90,7 +90,7 @@ export class CompletionSource {
             source: source
         };
 
-        return await this.jediFactory.getJediProxyHandler<proxy.ICompletionResult>(document.uri).sendCommand(cmd, token);
+        return this.jediFactory.getJediProxyHandler<proxy.ICompletionResult>(document.uri).sendCommand(cmd, token);
     }
 
     private toVsCodeCompletions(documentPosition: DocumentPosition, data: proxy.ICompletionResult, resource: vscode.Uri): vscode.CompletionItem[] {

src/client/providers/providerUtilities.ts

@@ -13,10 +13,19 @@ export function getDocumentTokens(document: vscode.TextDocument, tokenizeTo: vsc
 export function isPositionInsideStringOrComment(document: vscode.TextDocument, position: vscode.Position): boolean {
     const tokenizeTo = position.translate(1, 0);
     const tokens = getDocumentTokens(document, tokenizeTo, TokenizerMode.CommentsAndStrings);
-    const index = tokens.getItemContaining(document.offsetAt(position));
+    const offset = document.offsetAt(position);
+    let index = tokens.getItemContaining(offset);
     if (index >= 0) {
         const token = tokens.getItemAt(index);
         return token.type === TokenType.String || token.type === TokenType.Comment;
     }
+    if (offset > 0) {
+        // In case position is at the every end of the comment or unterminated string
+        index = tokens.getItemContaining(offset - 1);
+        if (index >= 0) {
+            const token = tokens.getItemAt(index);
+            return token.end === offset && token.type === TokenType.Comment;
+        }
+    }
     return false;
 }

src/test/autocomplete/base.test.ts

@@ -195,10 +195,11 @@ suite('Autocomplete', () => {
             new vscode.Position(3, 0),  // false
             new vscode.Position(4, 2),  // false
             new vscode.Position(4, 8),  // false
-            new vscode.Position(5, 4)   // false
+            new vscode.Position(5, 4),  // false
+            new vscode.Position(5, 10)  // false
         ];
         const expected = [
-            false, true, false, false, false, false, false, false, false, false
+            false, true, false, false, false, false, false, false, false, false, false
         ];
         const textDocument = await vscode.workspace.openTextDocument(fileSuppress);
         await vscode.window.showTextDocument(textDocument);

src/test/language/tokenizer.test.ts

@@ -180,15 +180,15 @@ suite('Language.Tokenizer', () => {
     });
     test('Operators', async () => {
         const text = '< <> << <<= ' +
-            '== != > >> >>= ' +
+            '== != > >> >>= >= <=' +
             '+ -' +
             '* ** / /= //=' +
             '*= += -= **= ' +
             '& &= | |= ^ ^=';
         const tokens = new Tokenizer().tokenize(text);
         const lengths = [
             1, 2, 2, 3,
-            2, 2, 1, 2, 3,
+            2, 2, 1, 2, 3, 2, 2,
             1, 1,
             1, 2, 1, 2, 3,
             2, 2, 2, 3,

tslint.json

@@ -59,6 +59,7 @@
         ],
         "no-unnecessary-type-assertion": false,
         "no-submodule-imports": false,
-        "no-redundant-jsdoc": false
+        "no-redundant-jsdoc": false,
+        "binary-expression-operand-order": false
     }
-}
+}