diff --git a/CHANGELOG.md b/CHANGELOG.md index 19f834186..22a4b5238 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,11 @@ - fixes currently-highlighted token in dark editor themes against light lab theme (and vice versa) ([#195][]) - restores sorting order-indicating caret icons in diagnostics panel table ([#261][]) + - handles document open and change operation ordering more predictably ([#284][]) [#195]: https://github.com/krassowski/jupyterlab-lsp/issues/195 [#261]: https://github.com/krassowski/jupyterlab-lsp/issues/261 +[#284]: https://github.com/krassowski/jupyterlab-lsp/pull/284 ### `@krassowski/jupyterlab-lsp 1.0.0` (2020-03-14) diff --git a/atest/02_Settings.robot b/atest/02_Settings.robot index 756bf7e12..0484c5254 100644 --- a/atest/02_Settings.robot +++ b/atest/02_Settings.robot @@ -4,6 +4,5 @@ Resource Keywords.robot *** Test Cases *** Settings - [Setup] Reset Application State Open in Advanced Settings ${LSP PLUGIN ID} Capture Page Screenshot 01-settings-lsp.png diff --git a/atest/04_Interface/DiagnosticsPanel.robot b/atest/04_Interface/DiagnosticsPanel.robot index 033a848b2..23f49aec3 100644 --- a/atest/04_Interface/DiagnosticsPanel.robot +++ b/atest/04_Interface/DiagnosticsPanel.robot @@ -1,6 +1,7 @@ *** Settings *** Suite Setup Setup Suite For Screenshots diagnostics_panel Resource ../Keywords.robot +Test Setup Gently Reset Workspace *** Variables *** ${EXPECTED_COUNT} 1 @@ -11,7 +12,6 @@ ${MENU COLUMN MESSAGE} xpath://div[contains(@class, 'p-Menu-itemLabel')][cont *** Test Cases *** Diagnostics Panel Opens - [Setup] Gently Reset Workspace Open Notebook And Panel Panel.ipynb Capture Page Screenshot 03-panel-opens.png Wait Until Keyword Succeeds 10 x 1s Should Have Expected Rows Count @@ -19,7 +19,6 @@ Diagnostics Panel Opens Diagnostics Panel Works After Rename [Documentation] Test for #141 bug (diagnostics were not cleared after rename) - [Setup] Gently Reset Workspace Open Notebook And Panel Panel.ipynb Rename Jupyter File Panel.ipynb PanelRenamed.ipynb Close Diagnostics Panel @@ -32,7 +31,6 @@ Diagnostics Panel Works After Rename [Teardown] Clean Up After Working With File Panel.ipynb Diagnostics Panel Can Be Restored - [Setup] Gently Reset Workspace Open Notebook And Panel Panel.ipynb Close Diagnostics Panel Open Diagnostics Panel @@ -40,7 +38,6 @@ Diagnostics Panel Can Be Restored [Teardown] Clean Up After Working With File Panel.ipynb Columns Can Be Hidden - [Setup] Gently Reset Workspace Open Notebook And Panel Panel.ipynb Wait Until Keyword Succeeds 10 x 1s Element Should Contain ${DIAGNOSTICS PANEL} ${DIAGNOSTIC MESSAGE} Open Context Menu Over css:.lsp-diagnostics-listing th diff --git a/atest/05_Features/Completion.robot b/atest/05_Features/Completion.robot index 4857cadaa..531542162 100644 --- a/atest/05_Features/Completion.robot +++ b/atest/05_Features/Completion.robot @@ -1,6 +1,6 @@ *** Settings *** Suite Setup Setup Suite For Screenshots completion -Test Setup Setup Notebook Python Completion.ipynb +Test Setup Setup Completion Test Test Teardown Clean Up After Working With File Completion.ipynb Force Tags feature:completion Resource ../Keywords.robot @@ -92,6 +92,9 @@ Triggers Completer On Dot Completer Should Suggest append *** Keywords *** +Setup Completion Test + Setup Notebook Python Completion.ipynb + Get Cell Editor Content [Arguments] ${cell_nr} ${content} Execute JavaScript return document.querySelector('.jp-Cell:nth-child(${cell_nr}) .CodeMirror').CodeMirror.getValue() diff --git a/atest/Keywords.robot b/atest/Keywords.robot index 287341473..055085b16 100644 --- a/atest/Keywords.robot +++ b/atest/Keywords.robot @@ -22,7 +22,9 @@ Setup Server and Browser Initialize User Settings ${cmd} = Create Lab Launch Command ${root} Set Screenshot Directory ${OUTPUT DIR}${/}screenshots - ${server} = Start Process ${cmd} shell=yes env:HOME=${home} cwd=${home} stdout=${OUTPUT DIR}${/}lab.log + Set Global Variable ${LAB LOG} ${OUTPUT DIR}${/}lab.log + Set Global Variable ${PREVIOUS LAB LOG LENGTH} 0 + ${server} = Start Process ${cmd} shell=yes env:HOME=${home} cwd=${home} stdout=${LAB LOG} ... stderr=STDOUT Set Global Variable ${SERVER} ${server} Open JupyterLab @@ -65,6 +67,14 @@ Tear Down Everything Terminate All Processes Terminate All Processes kill=${True} +Lab Log Should Not Contain Known Error Messages + ${log} = Get File ${LAB LOG} + ${test log} = Set Variable ${log[${PREVIOUS LAB LOG LENGTH}:]} + ${length} = Get Length ${log} + Set Global Variable ${PREVIOUS LAB LOG LENGTH} ${length} + Run Keyword If ("${OS}", "${PY}") !\= ("Windows", "36") + ... Should Not Contain Any ${test log} @{KNOWN BAD ERRORS} + Wait For Splash Go To ${URL}lab?reset&token=${TOKEN} Set Window Size 1024 768 @@ -201,6 +211,7 @@ Clean Up After Working With File [Arguments] ${file} Remove File ${OUTPUT DIR}${/}home${/}${file} Reset Application State + Lab Log Should Not Contain Known Error Messages Setup Notebook [Arguments] ${Language} ${file} ${isolated}=${True} diff --git a/atest/Variables.robot b/atest/Variables.robot index d0535ebd2..17006e7c3 100644 --- a/atest/Variables.robot +++ b/atest/Variables.robot @@ -37,3 +37,7 @@ ${LSP PLUGIN SETTINGS FILE} @krassowski${/}jupyterlab-lsp${/}plugin.jupyterla ${CSS USER SETTINGS} .jp-SettingsRawEditor-user # diagnostics ${CSS DIAGNOSTIC} css:.cm-lsp-diagnostic +# log messages +@{KNOWN BAD ERRORS} +... pyls_jsonrpc.endpoint - Failed to handle notification +... pyls_jsonrpc.endpoint - Failed to handle request diff --git a/atest/__init__.robot b/atest/__init__.robot index 56d83a5a5..4cc357221 100644 --- a/atest/__init__.robot +++ b/atest/__init__.robot @@ -2,5 +2,6 @@ Suite Setup Setup Server and Browser Suite Teardown Tear Down Everything Test Setup Reset Application State +Test Teardown Lab Log Should Not Contain Known Error Messages Force Tags os:${OS.lower()} py:${PY} Resource Keywords.robot diff --git a/ci/env-test.yml.in b/ci/env-test.yml.in index 938a1dbc0..ef37eb323 100644 --- a/ci/env-test.yml.in +++ b/ci/env-test.yml.in @@ -15,9 +15,11 @@ dependencies: - python-language-server - ujson <=1.35 # for R language server and kernel - - r + # TODO: try r 4.0 soon + - r <4 - r-irkernel - r-languageserver + - r-stringi >=1.4.6 - rpy2 # test tools - pytest-asyncio diff --git a/ci/job.test.yml b/ci/job.test.yml index 6801842ee..e12289aaf 100644 --- a/ci/job.test.yml +++ b/ci/job.test.yml @@ -2,30 +2,30 @@ parameters: platforms: - name: Linux vmImage: ubuntu-16.04 - activate: source activate + activate: source activate base - name: MacOSX vmImage: macos-10.14 - activate: source activate + activate: source activate base - name: Windows vmImage: vs2017-win2016 - activate: call activate + activate: call activate base pythons: - name: ThreeSix - spec: '>=3.6,<3.7.0a0' + spec: '=3.6' lab: '>=2,<3.0.0a0' nodejs: '>=10,<11.0.0.a0' - name: ThreeSeven - spec: '>=3.7,<3.8.0a0' + spec: '=3.7' lab: '>=2,<3.0.0a0' nodejs: '>=12,<13.0.0a0' - name: ThreeEight - spec: '>=3.8,<3.9.0a0' + spec: '=3.8' lab: '>=2,<3.0.0a0' nodejs: '>=13,<14.0.0a0' js_cov_packages: - jupyterlab-go-to-definition - jupyterlab-lsp - env_update: conda env update -n jupyterlab-lsp --file env-test.yml --quiet + env_update: conda env update -n base --file env-test.yml --quiet lab_link: jupyter labextension link --debug --no-build $(LINKED_EXTENSIONS) lab_ext: jupyter labextension install --debug --no-build $(FIRST_PARTY_LABEXTENSIONS) lab_build: jupyter lab build --debug --dev-build=False --minimize=True @@ -40,6 +40,7 @@ jobs: - template: steps.conda.yml parameters: name: ${{ platform.name }} + packages: "'python${{ python.spec }}'" - script: ${{ platform.activate }} && cd ci && python env_template.py "${{ python.spec }}" "${{ python.lab }}" "${{ python.nodejs }}" displayName: generate env with python, lab, and nodejs version @@ -47,25 +48,28 @@ jobs: - script: ${{ parameters.env_update }} || ${{ parameters.env_update }} || ${{ parameters.env_update }} displayName: update conda environment with test dependencies - - script: conda info && conda list -n jupyterlab-lsp + - script: conda info && conda list -n base displayName: list conda packages and info - - script: ${{ platform.activate }} jupyterlab-lsp && jlpm || jlpm || jlpm + - script: ${{ platform.activate }} && jlpm || jlpm || jlpm displayName: install npm dependencies - - script: ${{ platform.activate }} jupyterlab-lsp && jlpm build + - script: ${{ platform.activate }} && jlpm build displayName: build typescript - - script: ${{ platform.activate }} jupyterlab-lsp && python setup.py sdist bdist_wheel + - script: ${{ platform.activate }} && python setup.py sdist bdist_wheel displayName: build python distributions - - script: ${{ platform.activate }} jupyterlab-lsp && jlpm lerna run bundle + - script: ${{ platform.activate }} && jlpm lerna run bundle displayName: build npm bundles - - script: ${{ platform.activate }} jupyterlab-lsp && cd dist && python -m pip install jupyter_lsp-$(PY_JLSP_VERSION)-py3-none-any.whl --no-deps + - script: ${{ platform.activate }} && cd dist && python -m pip install jupyter_lsp-$(PY_JLSP_VERSION)-py3-none-any.whl --no-deps displayName: install python wheel - - script: ${{ platform.activate }} jupyterlab-lsp && jlpm test + - script: ${{ platform.activate }} && python scripts/jedi_cache.py + displayName: warm up jedi cache + + - script: ${{ platform.activate }} && jlpm test displayName: run frontend unit tests - task: PublishTestResults@2 @@ -84,28 +88,28 @@ jobs: summaryFileLocation: 'packages/${{ js_package }}/coverage/cobertura-coverage.xml' condition: always() - - script: ${{ platform.activate }} jupyterlab-lsp && jupyter serverextension list + - script: ${{ platform.activate }} && jupyter serverextension list displayName: list server extensions - - script: ${{ platform.activate }} jupyterlab-lsp && python scripts/utest.py --test-run-title="Pytest ${{ platform.name }}${{ python.name }}" + - script: ${{ platform.activate }} && python scripts/utest.py --test-run-title="Pytest ${{ platform.name }}${{ python.name }}" displayName: run python tests - - script: ${{ platform.activate }} jupyterlab-lsp && ${{ parameters.lab_link }} || ${{ parameters.lab_link }} || ${{ parameters.lab_link }} + - script: ${{ platform.activate }} && ${{ parameters.lab_link }} || ${{ parameters.lab_link }} || ${{ parameters.lab_link }} displayName: install support packages - - script: ${{ platform.activate }} jupyterlab-lsp && ${{ parameters.lab_ext }} || ${{ parameters.lab_ext }} || ${{ parameters.lab_ext }} + - script: ${{ platform.activate }} && ${{ parameters.lab_ext }} || ${{ parameters.lab_ext }} || ${{ parameters.lab_ext }} displayName: install labextensions - - script: ${{ platform.activate }} jupyterlab-lsp && jupyter labextension list + - script: ${{ platform.activate }} && jupyter labextension list displayName: list labextensions before build - - script: ${{ platform.activate }} jupyterlab-lsp && ${{ parameters.lab_build }} || ${{ parameters.lab_build }} || ${{ parameters.lab_build }} + - script: ${{ platform.activate }} && ${{ parameters.lab_build }} || ${{ parameters.lab_build }} || ${{ parameters.lab_build }} displayName: build lab - - script: ${{ platform.activate }} jupyterlab-lsp && jupyter labextension list + - script: ${{ platform.activate }} && jupyter labextension list displayName: list labextensions after build - - script: ${{ platform.activate }} jupyterlab-lsp && python scripts/atest.py + - script: ${{ platform.activate }} && python scripts/atest.py displayName: run browser tests - task: PublishTestResults@2 diff --git a/packages/jupyterlab-lsp/src/connection.ts b/packages/jupyterlab-lsp/src/connection.ts index 005b57d27..d73d0c971 100644 --- a/packages/jupyterlab-lsp/src/connection.ts +++ b/packages/jupyterlab-lsp/src/connection.ts @@ -119,9 +119,12 @@ export class LSPConnection extends LspWsConnection { changeEvents: lsProtocol.TextDocumentContentChangeEvent[], documentInfo: IDocumentInfo ) { - if (!this.isConnected || !this.isInitialized) { + if (!this.isReady) { return; } + if (!this.openedUris.get(documentInfo.uri)) { + this.sendOpen(documentInfo); + } const textDocumentChange: lsProtocol.DidChangeTextDocumentParams = { textDocument: { uri: documentInfo.uri, diff --git a/packages/lsp-ws-connection/src/ws-connection.ts b/packages/lsp-ws-connection/src/ws-connection.ts index 0cd60391a..24acea4eb 100644 --- a/packages/lsp-ws-connection/src/ws-connection.ts +++ b/packages/lsp-ws-connection/src/ws-connection.ts @@ -32,6 +32,7 @@ export class LspWsConnection extends events.EventEmitter public serverCapabilities: protocol.ServerCapabilities; protected socket: WebSocket; protected connection: MessageConnection; + protected openedUris = new Map(); private rootUri: string; constructor(options: ILspOptions) { @@ -129,6 +130,7 @@ export class LspWsConnection extends events.EventEmitter if (this.connection) { this.connection.dispose(); } + this.openedUris.clear(); this.socket.close(); } @@ -202,6 +204,8 @@ export class LspWsConnection extends events.EventEmitter return; } + this.openedUris.clear(); + const message: protocol.InitializeParams = this.initializeParams(); this.connection @@ -229,6 +233,7 @@ export class LspWsConnection extends events.EventEmitter 'textDocument/didOpen', textDocumentMessage ); + this.openedUris.set(documentInfo.uri, true); this.sendChange(documentInfo); } @@ -236,6 +241,10 @@ export class LspWsConnection extends events.EventEmitter if (!this.isReady) { return; } + if (!this.openedUris.get(documentInfo.uri)) { + this.sendOpen(documentInfo); + return; + } const textDocumentChange: protocol.DidChangeTextDocumentParams = { textDocument: { uri: documentInfo.uri, diff --git a/py_src/jupyter_lsp/specs/pyls.py b/py_src/jupyter_lsp/specs/pyls.py index d74070dc8..4dcaafafd 100644 --- a/py_src/jupyter_lsp/specs/pyls.py +++ b/py_src/jupyter_lsp/specs/pyls.py @@ -1,9 +1,9 @@ from .config import load_config_schema -from .utils import ShellSpec +from .utils import PythonModuleSpec -class PythonLanguageServer(ShellSpec): - key = cmd = "pyls" +class PythonLanguageServer(PythonModuleSpec): + python_module = key = "pyls" languages = ["python"] spec = dict( display_name="pyls", diff --git a/py_src/jupyter_lsp/specs/utils.py b/py_src/jupyter_lsp/specs/utils.py index e8c4b67cd..aaa5a8d3a 100644 --- a/py_src/jupyter_lsp/specs/utils.py +++ b/py_src/jupyter_lsp/specs/utils.py @@ -1,5 +1,6 @@ import os import shutil +import sys from pathlib import Path from typing import List, Text @@ -32,7 +33,7 @@ def __call__( return {} -class ShellSpec(SpecBase): +class ShellSpec(SpecBase): # pragma: no cover """ Helper for a language server spec for executables on $PATH in the notebook server environment. """ @@ -61,6 +62,29 @@ def __call__(self, mgr: LanguageServerManagerAPI) -> KeyedLanguageServerSpecs: } +class PythonModuleSpec(SpecBase): + """ Helper for a python-based language server spec in the notebook server + environment + """ + + python_module = "" + + def __call__(self, mgr: LanguageServerManagerAPI) -> KeyedLanguageServerSpecs: + spec = __import__("importlib").util.find_spec(self.python_module) + + if not spec.origin: # pragma: no cover + return {} + + return { + self.key: { + "argv": [sys.executable, "-m", self.python_module, *self.args], + "languages": self.languages, + "version": SPEC_VERSION, + **self.spec, + } + } + + class NodeModuleSpec(SpecBase): """ Helper for a nodejs-based language server spec in one of several node_modules diff --git a/scripts/atest.py b/scripts/atest.py index aefbeb63a..b11a89fdc 100644 --- a/scripts/atest.py +++ b/scripts/atest.py @@ -20,12 +20,16 @@ OS_PY_ARGS = { # notebook and ipykernel releases do not yet support python 3.8 on windows # ("Windows", "38"): ["--include", "not-supported", "--runemptysuite"] + # TODO: restore when we figure out win36 vs jedi on windows + ("Windows", "36"): ["--exclude", "feature:completion", "--runemptysuite"] } NON_CRITICAL = [ # TODO: restore when yaml-language-server supports both config and... # everything else: https://github.com/krassowski/jupyterlab-lsp/pull/245 - ["language:yaml", "feature:config"] + ["language:yaml", "feature:config"], + # TODO: restore when we figure out win36 vs jedi on windows + ["language:python", "py:36", "os:windows"], ] diff --git a/scripts/jedi_cache.py b/scripts/jedi_cache.py new file mode 100644 index 000000000..998e14963 --- /dev/null +++ b/scripts/jedi_cache.py @@ -0,0 +1,118 @@ +""" utility script to warm up/validate the jedi cache +what does it do: +- Deletes the jedi cache (usually already empty on CI) +- Imports a bunch of libraries +- Prints out some versions, especially ones that are + at times troublesome +- Forces indexing all of the loaded libraries and their + dependencies + +why is this needed: +- Before we had this, a couple of browser tests appeared + "consistently flakier" than they were, as they were + time-bounded by creating the jedi cache. +- This was taking up to a minute to get a single + completion value back +- Further, were this cache to get corrupted (perhaps due to + killing a running test :P) things go very mysteriously bad. +- We need a way for the cache to be right before testing + +how does it work: +- When _using_ jedi for the first time, the cache gets + created in `jedi.settings.cache_directory`, usually + somewhere in $HOME. +- As different libriraries are inspected by jedi, they get + added to the cache. +- This is very slow, especially on windows, and cannot + feasibly be cached, today. +- This script accelerates this process, so it can be done + in a controlled manner rather than during some other test +- also, by running it ahead of time, if the jedi dependency + chain is broken in any way, this should help determine + if faster, before trying to build everything + +see more: +- https://jedi.readthedocs.io/en/latest/docs/settings.html +- https://github.com/krassowski/jupyterlab-lsp/pull/284 + +""" +import os +import pathlib +import shutil +import sys +import time + +import IPython +import jedi +import jupyterlab +import notebook +import parso +import pyls + +SOURCE_TEMPLATE = """ +import {module} +{module}.""" + +MODULES_TO_CACHE = [ + "itertools", + "statistics", + *sys.modules, +] + +ENV = jedi.InterpreterEnvironment() + + +def warm_up_one(module): + print(module, end="\t") + start = time.time() + script = jedi.Script(SOURCE_TEMPLATE.format(module=module), environment=ENV) + completions = len(script.complete(3, len("{}.".format(module)))) + end = time.time() + print("\t", completions, end - start) + + +def print_versions(): + print(".".join(map(str, sys.version_info[:3])), "\t", "python") + print(IPython.__version__, "\t", "ipython") + print(jedi.__version__, "\t", "jedi") + print(jupyterlab.__version__, "\t", "jupyterlab") + print(notebook.__version__, "\t", "notebook") + print(parso.__version__, "\t", "parso") + print(pyls.__version__, "\t", "pyls") + + +def print_env(): + [ + print(key, "\t", value) + for key, value in sorted(os.environ.items()) + if "CONDA" in key + ] + + +def setup_jedi(): + print("default jedi environment", jedi.api.environment.get_default_environment()) + print("jedi environment", ENV) + jedi_cache = pathlib.Path(jedi.settings.cache_directory) + if jedi_cache.exists(): + shutil.rmtree(jedi_cache) + print("removed jedi cache!") + else: + print("no jedi cache was found!") + + +def warm_up(modules): + print_env() + print("-" * 80) + print_versions() + print("-" * 80) + setup_jedi() + print("-" * 80) + start = time.time() + [warm_up_one(module) for module in modules] + end = time.time() + print("-" * 80) + print(len(modules), "modules in", jedi.settings.cache_directory, end - start) + + +if __name__ == "__main__": + warm_up(sorted(set(sys.argv[1:] or MODULES_TO_CACHE)))