Refactor LSPConnection, ConnectionManager

.gitignore

@@ -114,3 +114,4 @@ junit.xml
 coverage/
 .vscode/
 _schema.d.ts
+.virtual_documents/

atest/01_Editor.robot

@@ -11,9 +11,7 @@ ${CM CURSORS}     css:.CodeMirror-cursors:not([style='visibility: hidden'])
 
 *** Test Cases ***
 Bash
-    [Documentation]    TODO: figure out why the first server is extra flaky
-    Wait Until Keyword Succeeds    6x    5s    Editor Shows Features for Language    Bash    example.sh    Diagnostics=Failed to parse expression
-    ...    Jump to Definition=fib
+    Editor Shows Features for Language    Bash    example.sh    Diagnostics=Failed to parse expression    Jump to Definition=fib
 
 CSS
     ${def} =    Set Variable    xpath:(//span[contains(@class, 'cm-variable-2')][contains(text(), '--some-var')])[last()]
@@ -67,7 +65,7 @@ Editor Shows Features for Language
     Set Tags    language:${Language.lower()}
     Set Screenshot Directory    ${OUTPUT DIR}${/}screenshots${/}editor${/}${Language.lower()}
     Copy File    examples${/}${file}    ${OUTPUT DIR}${/}home${/}${file}
-    Lab Command    Close All Tabs
+    Try to Close All Tabs
     Open ${file} in ${MENU EDITOR}
     Capture Page Screenshot    00-opened.png
     FOR    ${f}    IN    @{features}

atest/03_Notebook.robot

@@ -1,10 +1,10 @@
 *** Settings ***
 Suite Setup       Setup Suite For Screenshots    notebook
+Test Setup        Try to Close All Tabs
 Resource          Keywords.robot
 
 *** Test Cases ***
 Python
-    [Setup]    Gently Reset Workspace
     Setup Notebook    Python    Python.ipynb
     Capture Page Screenshot    01-python.png
     ${diagnostic} =    Set Variable    W291 trailing whitespace (pycodestyle)
@@ -13,11 +13,11 @@ Python
     Clean Up After Working With File    Python.ipynb
 
 Foregin Extractors
-    [Setup]    Gently Reset Workspace
     Setup Notebook    Python    Foreign extractors.ipynb
-    @{diagnostics} =    Create List    Failed to parse expression    undefined name 'valid' (pyflakes)    Trailing whitespace is superfluous. (lintr)
+    # if mypy and pyflakes will fight over `(N|n)ame 'valid'`, just hope for the best
+    @{diagnostics} =    Create List    Failed to parse expression    ame 'valid'    Trailing whitespace is superfluous. (lintr)
     FOR    ${diagnostic}    IN    @{diagnostics}
-        Wait Until Page Contains Element    css:.cm-lsp-diagnostic[title="${diagnostic}"]    timeout=35s
+        Wait Until Page Contains Element    css:.cm-lsp-diagnostic[title*\="${diagnostic}"]    timeout=35s
         Capture Page Screenshot    0x-${diagnostic}.png
     END
     Clean Up After Working With File    Foreign Extractors.ipynb

atest/05_Features/Completion.robot

@@ -11,7 +11,6 @@ Works With Kernel Running
     [Documentation]    The suggestions from kernel and LSP should get integrated.
     [Tags]    language:python
     Setup Notebook    Python    Completion.ipynb
-    Wait Until Fully Initialized
     Enter Cell Editor    1    line=2
     Capture Page Screenshot    01-entered-cell.png
     Trigger Completer
@@ -31,7 +30,6 @@ Works With Kernel Running
 Works When Kernel Is Shut Down
     [Tags]    language:python
     Setup Notebook    Python    Completion.ipynb
-    Wait Until Fully Initialized
     Lab Command    Shut Down All Kernels…
     Capture Page Screenshot    01-shutting-kernels.png
     Accept Default Dialog Option
@@ -59,7 +57,6 @@ User Can Select Lowercase After Starting Uppercase
     [Tags]    language:python
     Setup Notebook    Python    Completion.ipynb
     # `from time import Tim<tab>` → `from time import time`
-    Wait Until Fully Initialized
     Enter Cell Editor    5    line=1
     Trigger Completer
     Completer Should Suggest    time
@@ -73,7 +70,6 @@ Mid Token Completions Do Not Overwrite
     # `disp<tab>data` → `display_table<cursor>data`
     Place Cursor In Cell Editor At    9    line=1    character=4
     Capture Page Screenshot    01-cursor-placed.png
-    Wait Until Fully Initialized
     Press Keys    None    TAB
     Capture Page Screenshot    02-completed.png
     Wait Until Keyword Succeeds    40x    0.5s    Cell Editor Should Equal    9    display_tabledata
@@ -88,7 +84,6 @@ Completion Works For Tokens Separated By Space
     Setup Notebook    Python    Completion.ipynb
     # `from statistics <tab>` → `from statistics import<cursor>`
     Enter Cell Editor    13    line=1
-    Wait Until Fully Initialized
     Trigger Completer
     Completer Should Suggest    import
     Press Keys    None    ENTER
@@ -102,7 +97,6 @@ Kernel And LSP Completions Merge Prefix Conflicts Are Resolved
     # `import os.pat<tab>` → `import os.pathsep`
     Setup Notebook    Python    Completion.ipynb
     Enter Cell Editor    15    line=1
-    Wait Until Fully Initialized
     Trigger Completer
     Completer Should Suggest    pathsep
     Select Completer Suggestion    pathsep
@@ -113,7 +107,6 @@ Triggers Completer On Dot
     [Tags]    language:python
     Setup Notebook    Python    Completion.ipynb
     Enter Cell Editor    2    line=1
-    Wait Until Fully Initialized
     Press Keys    None    .
     Wait Until Keyword Succeeds    10x    0.5s    Cell Editor Should Equal    2    list.
     Wait Until Page Contains Element    ${COMPLETER_BOX}    timeout=35s

atest/05_Features/Signature.robot

@@ -9,7 +9,6 @@ ${SIGNATURE_BOX}    css:.lsp-signature-help
 *** Test Cases ***
 Triggers Signature Help After A Keystroke
     Setup Notebook    Python    Signature.ipynb
-    Wait Until Fully Initialized
     Enter Cell Editor    1    line=6
     Capture Page Screenshot    01-entered-cell.png
     Press Keys    None    (

atest/Keywords.robot

@@ -66,9 +66,17 @@ Open JupyterLab
 Close JupyterLab
     Close All Browsers
 
-Reset Application State
+Close All Tabs
+    Accept Default Dialog Option
     Lab Command    Close All Tabs
     Accept Default Dialog Option
+
+Try to Close All Tabs
+    Wait Until Keyword Succeeds    5x    50ms    Close All Tabs
+
+Reset Application State
+    Try to Close All Tabs
+    Accept Default Dialog Option
     Ensure All Kernels Are Shut Down
     Lab Command    Reset Application State
     Wait For Splash
@@ -174,9 +182,11 @@ Setup Notebook
     Set Tags    language:${Language.lower()}
     Set Screenshot Directory    ${OUTPUT DIR}${/}screenshots${/}notebook${/}${file.lower()}
     Copy File    examples${/}${file}    ${OUTPUT DIR}${/}home${/}${file}
-    Lab Command    Close All Tabs
+    Try to Close All Tabs
     Open ${file} in ${MENU NOTEBOOK}
+    Wait Until Fully Initialized
     Capture Page Screenshot    00-opened.png
+    Capture Page Screenshot    01-initialized.png
 
 Open Diagnostics Panel
     Lab Command    Show Diagnostics Panel
@@ -194,7 +204,7 @@ Wait For Dialog
     Wait Until Page Contains Element    ${DIALOG WINDOW}    timeout=180s
 
 Gently Reset Workspace
-    Lab Command    Close All Tabs
+    Try to Close All Tabs
 
 Enter Cell Editor
     [Arguments]    ${cell_nr}    ${line}=1
@@ -207,7 +217,7 @@ Place Cursor In Cell Editor At
     Execute JavaScript    return document.querySelector('.jp-Cell:nth-child(${cell_nr}) .CodeMirror').CodeMirror.setCursor({line: ${line} - 1, ch: ${character}})
 
 Wait Until Fully Initialized
-    Wait Until Element Contains    ${STATUSBAR}    Fully initialized    timeout=35s
+    Wait Until Element Contains    ${STATUSBAR}    Fully initialized    timeout=60s
 
 Open Context Menu Over
     [Arguments]    ${sel}

azure-pipelines.yml

@@ -22,6 +22,9 @@ variables:
   THIRD_PARTY_LABEXTENSIONS: >-
     @krassowski/jupyterlab_go_to_definition
 
+  LINKED_EXTENSIONS: >-
+    packages/lsp-ws-connection
+
 jobs:
   - template: ci/job.lint.yml
   - template: ci/job.test.yml

ci/job.test.yml

@@ -20,6 +20,7 @@ parameters:
       spec: '>=3.8,<3.9.0a0'
       lab: '>=1.2.4,<1.3.0a0'
   env_update: conda env update -n jupyterlab-lsp --file env-test.yml --quiet
+  lab_link: jupyter labextension link --no-build $(LINKED_EXTENSIONS)
   lab_ext: jupyter labextension install --no-build $(THIRD_PARTY_LABEXTENSIONS) $(FIRST_PARTY_LABEXTENSIONS)
 
 jobs:
@@ -79,6 +80,9 @@ jobs:
               - script: ${{ platform.activate }} jupyterlab-lsp && python scripts/utest.py --test-run-title="Pytest ${{ platform.name }}${{ python.name }}"
                 displayName: run python tests
 
+              - script: ${{ platform.activate }} jupyterlab-lsp && ${{ parameters.lab_link }} || ${{ parameters.lab_link }} || ${{ parameters.lab_link }}
+                displayName: install support packages
+
               - script: ${{ platform.activate }} jupyterlab-lsp && ${{ parameters.lab_ext }} || ${{ parameters.lab_ext }} || ${{ parameters.lab_ext }}
                 displayName: install labextensions
 

packages/jupyterlab-lsp/src/adapters/codemirror/cm_adapter.ts

@@ -38,19 +38,28 @@ export class CodeMirrorAdapter {
 
   public async updateAfterChange() {
     this.jupyterlab_components.remove_tooltip();
-    await until_ready(() => this.last_change != null, 30, 22).catch(() => {
-      this.invalidateLastChange();
-      throw Error(
+
+    try {
+      await until_ready(() => this.last_change != null, 30, 22);
+    } catch (err) {
+      console.log(
         'No change obtained from CodeMirror editor within the expected time of 0.66s'
       );
-    });
+      return;
+    }
+
     let change: CodeMirror.EditorChange = this.last_change;
 
+    let root_position: IRootPosition;
+
     try {
-      const root_position = this.editor
-        .getDoc()
-        .getCursor('end') as IRootPosition;
+      root_position = this.editor.getDoc().getCursor('end') as IRootPosition;
+    } catch (err) {
+      console.log('LSP: Root positon not found');
+      return;
+    }
 
+    try {
       let document = this.editor.document_at_root_position(root_position);
 
       if (this.virtual_document !== document) {

packages/jupyterlab-lsp/src/adapters/codemirror/feature.spec.ts

@@ -21,6 +21,7 @@ import { nbformat } from '@jupyterlab/coreutils';
 import { language_specific_overrides } from '../../magics/defaults';
 import { foreign_code_extractors } from '../../extractors/defaults';
 import { NotebookModel } from '@jupyterlab/notebook';
+import { PageConfig } from '@jupyterlab/coreutils';
 
 const js_fib_code = `function fib(n) {
   return n<2?n:fib(n-1)+fib(n-2);
@@ -94,6 +95,8 @@ const js_partial_edits = [
 ];
 
 describe('Feature', () => {
+  PageConfig.setOption('rootUri', 'file://');
+
   describe('apply_edit()', () => {
     class EditApplyingFeature extends CodeMirrorLSPFeature {
       name = 'EditApplyingFeature';
@@ -248,9 +251,9 @@ describe('Feature', () => {
           'def a_function_2():\n    pass\n\n\nx = a_function_2()\n';
         expect(main_document.value).to.be.equal(old_virtual_source);
 
-        let result = await feature.do_apply_edit({
+        let outcome = await feature.do_apply_edit({
           changes: {
-            ['file://' + environment.path()]: [
+            [main_document.document_info.uri]: [
               {
                 range: {
                   start: { line: 0, character: 0 },
@@ -261,6 +264,7 @@ describe('Feature', () => {
             ]
           }
         });
+
         await synchronizeContent();
 
         let value = main_document.value;
@@ -274,9 +278,9 @@ describe('Feature', () => {
         );
         expect(code_cells[1]).to.have.property('source', 'x = a_function_2()');
 
-        expect(result.appliedChanges).to.be.equal(null);
-        expect(result.wasGranular).to.be.equal(false);
-        expect(result.modifiedCells).to.be.equal(2);
+        expect(outcome.appliedChanges).to.be.equal(null);
+        expect(outcome.wasGranular).to.be.equal(false);
+        expect(outcome.modifiedCells).to.be.equal(2);
       });
 
       it('handles IPython magics', async () => {
@@ -317,7 +321,7 @@ print(x)""")
 
         await feature.do_apply_edit({
           changes: {
-            ['file://' + environment.path()]: [
+            [main_document.document_info.uri]: [
               {
                 range: {
                   start: { line: 0, character: 0 },

packages/jupyterlab-lsp/src/adapters/codemirror/feature.ts

@@ -194,7 +194,7 @@ export abstract class CodeMirrorLSPFeature implements ILSPFeature {
     let start = PositionConverter.lsp_to_cm(range.start) as IVirtualPosition;
     let end = PositionConverter.lsp_to_cm(range.end) as IVirtualPosition;
 
-    if (typeof cm_editor === 'undefined') {
+    if (cm_editor == null) {
       let start_in_root = this.transform_virtual_position_to_root_position(
         start
       );
@@ -278,7 +278,8 @@ export abstract class CodeMirrorLSPFeature implements ILSPFeature {
   protected async apply_edit(
     workspaceEdit: lsProtocol.WorkspaceEdit
   ): Promise<IEditOutcome> {
-    let current_uri = this.connection.getDocumentUri();
+    let current_uri = this.virtual_document.document_info.uri;
+
     // Specs: documentChanges are preferred over changes
     let changes = workspaceEdit.documentChanges
       ? workspaceEdit.documentChanges.map(
@@ -292,6 +293,7 @@ export abstract class CodeMirrorLSPFeature implements ILSPFeature {
 
     for (let change of changes) {
       let uri = change.textDocument.uri;
+
       if (
         decodeURI(uri) !== decodeURI(current_uri) &&
         decodeURI(uri) !== '/' + decodeURI(current_uri)

packages/jupyterlab-lsp/src/adapters/codemirror/features/completion.ts

@@ -8,7 +8,7 @@ export class Completion extends CodeMirrorLSPFeature {
 
   get completionCharacters() {
     if (
-      typeof this._completionCharacters === 'undefined' ||
+      this._completionCharacters == null ||
       !this._completionCharacters.length
     ) {
       this._completionCharacters = this.connection.getLanguageCompletionCharacters();

packages/jupyterlab-lsp/src/adapters/codemirror/features/diagnostics.ts

@@ -2,7 +2,7 @@ import * as CodeMirror from 'codemirror';
 import * as lsProtocol from 'vscode-languageserver-protocol';
 import { Menu } from '@phosphor/widgets';
 import { PositionConverter } from '../../../converter';
-import { IVirtualPosition } from '../../../positioning';
+import { IVirtualPosition, IEditorPosition } from '../../../positioning';
 import { diagnosticSeverityNames } from '../../../lsp';
 import { DefaultMap } from '../../../utils';
 import { CodeMirrorLSPFeature, IFeatureCommand } from '../feature';
@@ -14,7 +14,6 @@ import {
   IEditorDiagnostic
 } from './diagnostics_listing';
 import { VirtualDocument } from '../../../virtual/document';
-import { DocumentConnectionManager } from '../../../connection_manager';
 import { VirtualEditor } from '../../../virtual/editor';
 
 // TODO: settings
@@ -118,7 +117,6 @@ export class Diagnostics extends CodeMirrorLSPFeature {
         }
 
         if (!panel_widget.isAttached) {
-          console.warn(adapter.widget_id);
           app.shell.add(panel_widget, 'main', {
             ref: adapter.widget_id,
             mode: 'split-bottom'
@@ -223,19 +221,13 @@ export class Diagnostics extends CodeMirrorLSPFeature {
   }
 
   public handleDiagnostic(response: lsProtocol.PublishDiagnosticsParams) {
+    if (response.uri !== this.virtual_document.document_info.uri) {
+      return;
+    }
     /* TODO: gutters */
     try {
       let diagnostics_list: IEditorDiagnostic[] = [];
 
-      let my_uri = DocumentConnectionManager.solve_uris(
-        this.virtual_document,
-        ''
-      ).document;
-
-      if (response.uri !== my_uri) {
-        return;
-      }
-
       // Note: no deep equal for Sets or Maps in JS
       const markers_to_retain: Set<string> = new Set();
 
@@ -311,7 +303,16 @@ export class Diagnostics extends CodeMirrorLSPFeature {
           let cm_editor = document.get_editor_at_virtual_line(start);
 
           let start_in_editor = document.transform_virtual_to_editor(start);
-          let end_in_editor = document.transform_virtual_to_editor(end);
+          let end_in_editor: IEditorPosition;
+
+          // some servers return strange positions for ends
+          try {
+            end_in_editor = document.transform_virtual_to_editor(end);
+          } catch (err) {
+            console.warn('LSP: Malformed range for diagnostic', end);
+            end_in_editor = { ...start_in_editor, ch: start_in_editor.ch + 1 };
+          }
+
           let range_in_editor = {
             start: start_in_editor,
             end: end_in_editor
@@ -408,7 +409,7 @@ export function message_without_code(diagnostic: lsProtocol.Diagnostic) {
   let message = diagnostic.message;
   let code_str = '' + diagnostic.code;
   if (
-    typeof diagnostic.code !== 'undefined' &&
+    diagnostic.code != null &&
     diagnostic.code !== '' &&
     message.startsWith(code_str + '')
   ) {