From d6a8394089a0d61554d8ca48159754a62e947da2 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 15 Feb 2020 16:34:15 -0500 Subject: [PATCH 01/17] start refactor of language server URLs --- docs/EXTENDING.md | 8 +-- py_src/jupyter_lsp/handlers.py | 32 +++++---- py_src/jupyter_lsp/manager.py | 72 ++++++++++++------- py_src/jupyter_lsp/schema/schema.json | 13 ++-- py_src/jupyter_lsp/session.py | 9 +-- py_src/jupyter_lsp/tests/conftest.py | 42 +++++------ py_src/jupyter_lsp/tests/listener.py | 2 +- py_src/jupyter_lsp/tests/test_listener.py | 24 ++++--- py_src/jupyter_lsp/tests/test_session.py | 33 ++++----- py_src/jupyter_lsp/types.py | 45 +++++++----- .../jupyter_lsp/virtual_documents_shadow.py | 2 +- 11 files changed, 159 insertions(+), 123 deletions(-) diff --git a/docs/EXTENDING.md b/docs/EXTENDING.md index be5e7d6e1..0df2f9c4d 100644 --- a/docs/EXTENDING.md +++ b/docs/EXTENDING.md @@ -82,9 +82,9 @@ from jupyter_lsp import lsp_message_listener def load_jupyter_server_extension(nbapp): @lsp_message_listener("all") - async def my_listener(scope, message, languages, manager): - print("received a {} {} message about {}".format( - scope, message["method"], languages + async def my_listener(scope, message, language_server, manager): + print("received a {} {} message from {}".format( + scope, message["method"], language_server )) ``` @@ -95,5 +95,5 @@ def load_jupyter_server_extension(nbapp): Fine-grained controls are available as part of the Python API. Pass these as named arguments to `lsp_message_listener`. -- `languages`: a regular expression of languages +- `language_server`: a regular expression of language servers - `method`: a regular expression of LSP JSON-RPC method names diff --git a/py_src/jupyter_lsp/handlers.py b/py_src/jupyter_lsp/handlers.py index 56662b36d..1a79d4162 100644 --- a/py_src/jupyter_lsp/handlers.py +++ b/py_src/jupyter_lsp/handlers.py @@ -21,20 +21,20 @@ class LanguageServerWebSocketHandler(WebSocketMixin, WebSocketHandler, BaseHandl """ Setup tornado websocket to route to language server sessions """ - language = None # type: Optional[Text] + language_server = None # type: Optional[Text] - def open(self, language): - self.language = language + def open(self, language_server): + self.language_server = language_server self.manager.subscribe(self) - self.log.debug("[{}] Opened a handler".format(self.language)) + self.log.debug("[{}] Opened a handler".format(self.language_server)) async def on_message(self, message): - self.log.debug("[{}] Handling a message".format(self.language)) + self.log.debug("[{}] Handling a message".format(self.language_server)) await self.manager.on_client_message(message, self) def on_close(self): self.manager.unsubscribe(self) - self.log.debug("[{}] Closed a handler".format(self.language)) + self.log.debug("[{}] Closed a handler".format(self.language_server)) class LanguageServersHandler(BaseHandler): @@ -52,11 +52,11 @@ def get(self): """ finish with the JSON representations of the sessions """ response = { - "version": 1, - "sessions": sorted( - [session.to_json() for session in self.manager.sessions.values()], - key=lambda session: session["spec"]["languages"], - ), + "version": 2, + "sessions": { + language_server: session.to_json() + for language_server, session in self.manager.sessions.items() + }, } errors = list(self.validator.iter_errors(response)) @@ -71,14 +71,18 @@ def add_handlers(nbapp): """ Add Language Server routes to the notebook server web application """ lsp_url = ujoin(nbapp.base_url, "lsp") - re_langs = "(?P.*)" + re_langservers = "(?P.*)" opts = {"manager": nbapp.language_server_manager} nbapp.web_app.add_handlers( ".*", [ - (lsp_url, LanguageServersHandler, opts), - (ujoin(lsp_url, re_langs), LanguageServerWebSocketHandler, opts), + (ujoin(lsp_url, "status"), LanguageServersHandler, opts), + ( + ujoin(lsp_url, "ws", re_langservers), + LanguageServerWebSocketHandler, + opts, + ), ], ) diff --git a/py_src/jupyter_lsp/manager.py b/py_src/jupyter_lsp/manager.py index 320401043..ec92fad9d 100644 --- a/py_src/jupyter_lsp/manager.py +++ b/py_src/jupyter_lsp/manager.py @@ -43,7 +43,7 @@ class LanguageServerManager(LanguageServerManagerAPI): sessions = Dict_( trait=Instance(LanguageServerSession), default_value={}, - help="sessions keyed by languages served", + help="sessions keyed by language server name", ) # type: Dict[Tuple[Text], LanguageServerSession] all_listeners = List_(trait=LoadableCallable).tag(config=True) @@ -80,18 +80,16 @@ def init_language_servers(self) -> None: # coalesce the servers, allowing a user to opt-out by specifying `[]` self.language_servers = { - key: spec - for key, spec in language_servers.items() - if spec.get("argv") and spec.get("languages") + key: spec for key, spec in language_servers.items() if spec.get("argv") } def init_sessions(self): """ create, but do not initialize all sessions """ sessions = {} - for spec in self.language_servers.values(): - sessions[tuple(sorted(spec["languages"]))] = LanguageServerSession( - spec=spec, parent=self + for language_server, spec in self.language_servers.items(): + sessions[language_server] = LanguageServerSession( + language_server=language_server, spec=spec, parent=self ) self.sessions = sessions @@ -121,37 +119,59 @@ def init_listeners(self): def subscribe(self, handler): """ subscribe a handler to session, or sta """ - sessions = [] - for languages, candidate_session in self.sessions.items(): - if handler.language in languages: - sessions.append(candidate_session) + session = self.sessions.get(handler.language_server) - if sessions: - for session in sessions: - session.handlers = set([handler]) | session.handlers + if session is None: + self.log.error( + "[{}] no session: handler subscription failed".format( + handler.language_server + ) + ) + return + + session.handlers = set([handler]) | session.handlers async def on_client_message(self, message, handler): - await self.wait_for_listeners(MessageScope.CLIENT, message, [handler.language]) + await self.wait_for_listeners( + MessageScope.CLIENT, message, handler.language_server + ) + session = self.sessions.get(handler.language_server) + + if session is None: + self.log.error( + "[{}] no session: client message dropped".format( + handler.language_server + ) + ) + return - for session in self.sessions_for_handler(handler): - session.write(message) + session.write(message) async def on_server_message(self, message, session): - await self.wait_for_listeners( - MessageScope.SERVER, message, session.spec["languages"] - ) + language_servers = [ + ls_key for ls_key, sess in self.sessions.items() if sess == session + ] + + for language_servers in language_servers: + await self.wait_for_listeners( + MessageScope.SERVER, message, language_servers + ) for handler in session.handlers: handler.write_message(message) def unsubscribe(self, handler): - for session in self.sessions_for_handler(handler): - session.handlers = [h for h in session.handlers if h != handler] + session = self.sessions.get(handler.language_server) + + if session is None: + self.log.error( + "[{}] no session: handler unsubscription failed".format( + handler.language_server + ) + ) + return - def sessions_for_handler(self, handler): - for session in self.sessions.values(): - if handler in session.handlers: - yield session + session.handlers = [h for h in session.handlers if h != handler] def _autodetect_language_servers(self): entry_points = [] diff --git a/py_src/jupyter_lsp/schema/schema.json b/py_src/jupyter_lsp/schema/schema.json index 3ea90b7e6..b238a82e1 100644 --- a/py_src/jupyter_lsp/schema/schema.json +++ b/py_src/jupyter_lsp/schema/schema.json @@ -8,7 +8,7 @@ }, "current-version": { "description": "which version of the spec this implements", - "enum": [1], + "enum": [2], "title": "Spec Schema Version", "type": "number" }, @@ -163,11 +163,16 @@ "servers-response": { "properties": { "sessions": { - "description": "a list of servers that are, could be, or were running", - "items": { + "description": "named server sessions that are, could be, or were running", + "patternProperties": { + ".*": { + "$ref": "#/definitions/session" + } + }, + "additionalProperties": { "$ref": "#/definitions/session" }, - "type": "array" + "type": "object" }, "version": { "$ref": "#/definitions/current-version" diff --git a/py_src/jupyter_lsp/session.py b/py_src/jupyter_lsp/session.py index 1651a3568..25e3ad3f3 100644 --- a/py_src/jupyter_lsp/session.py +++ b/py_src/jupyter_lsp/session.py @@ -10,7 +10,7 @@ from tornado.queues import Queue from tornado.websocket import WebSocketHandler -from traitlets import Bunch, Instance, Set, UseEnum, observe +from traitlets import Bunch, Instance, Set, Unicode, UseEnum, observe from traitlets.config import LoggingConfigurable from . import stdio @@ -26,6 +26,7 @@ class LanguageServerSession(LoggingConfigurable): """ Manage a session for a connection to a language server """ + language_server = Unicode(help="the language server implementation name") spec = Schema(LANGUAGE_SERVER_SPEC) # run-time specifics @@ -60,9 +61,9 @@ def __init__(self, *args, **kwargs): atexit.register(self.stop) def __repr__(self): # pragma: no cover - return "".format( - **self.spec - ) + return ( + "" + ).format(language_server=self.language_server, **self.spec) def to_json(self): return dict( diff --git a/py_src/jupyter_lsp/tests/conftest.py b/py_src/jupyter_lsp/tests/conftest.py index a6ca1d7a2..a0da2d030 100644 --- a/py_src/jupyter_lsp/tests/conftest.py +++ b/py_src/jupyter_lsp/tests/conftest.py @@ -12,31 +12,25 @@ from jupyter_lsp.handlers import LanguageServersHandler, LanguageServerWebSocketHandler # these should always be available in a test environment () -KNOWN_LANGUAGES = [ - "bash", - "css", - "dockerfile", - "html", - "ipythongfm", - "javascript", - "json", - "jsx", - "less", - "markdown", - "python", - "scss", - "typescript-jsx", - "typescript", - "yaml", +KNOWN_SERVERS = [ + "bash-language-server", + "vscode-css-languageserver-bin", + "dockerfile-language-server-nodejs", + "vscode-html-languageserver-bin", + "unified-language-server", + "javascript-typescript-langserver", + "vscode-json-languageserver-bin", + "pyls", + "yaml-language-server", ] -CMD_BASED_LANGUAGES = {"Rscript": ["r"]} +CMD_BASED_SERVERS = {"Rscript": ["r-languageserver"]} -KNOWN_LANGUAGES += sum( - [langs for cmd, langs in CMD_BASED_LANGUAGES.items() if shutil.which(cmd)], [] +KNOWN_SERVERS += sum( + [langs for cmd, langs in CMD_BASED_SERVERS.items() if shutil.which(cmd)], [] ) -KNOWN_UNKNOWN_LANGUAGES = ["cobol"] +KNOWN_UNKNOWN_SERVERS = ["foo-language-server"] @fixture @@ -44,13 +38,13 @@ def manager() -> LanguageServerManager: return LanguageServerManager() -@fixture(params=sorted(KNOWN_LANGUAGES)) -def known_language(request): +@fixture(params=sorted(KNOWN_SERVERS)) +def known_server(request): return request.param -@fixture(params=sorted(KNOWN_UNKNOWN_LANGUAGES)) -def known_unknown_language(request): +@fixture(params=sorted(KNOWN_UNKNOWN_SERVERS)) +def known_unknown_server(request): return request.param diff --git a/py_src/jupyter_lsp/tests/listener.py b/py_src/jupyter_lsp/tests/listener.py index c4ebc5293..dd043c64f 100644 --- a/py_src/jupyter_lsp/tests/listener.py +++ b/py_src/jupyter_lsp/tests/listener.py @@ -1,2 +1,2 @@ -async def dummy_listener(scope, message, languages, manager): +async def dummy_listener(scope, message, language_server, manager): pass diff --git a/py_src/jupyter_lsp/tests/test_listener.py b/py_src/jupyter_lsp/tests/test_listener.py index d2e02c98a..84bce9341 100644 --- a/py_src/jupyter_lsp/tests/test_listener.py +++ b/py_src/jupyter_lsp/tests/test_listener.py @@ -19,7 +19,7 @@ async def test_listener_bad_traitlets(bad_string, handlers): @pytest.mark.asyncio -async def test_listeners(known_language, handlers, jsonrpc_init_msg): +async def test_listeners(known_server, handlers, jsonrpc_init_msg): """ will some listeners listen? """ handler, ws_handler = handlers @@ -36,7 +36,7 @@ async def test_listeners(known_language, handlers, jsonrpc_init_msg): assert re.match( ( "," - " method=None, language=None>" + " method=None, language_server=None>" ), repr(dummy_listener), ) @@ -46,39 +46,41 @@ async def test_listeners(known_language, handlers, jsonrpc_init_msg): all_listened = Queue() # some client listeners - @lsp_message_listener("client", language=known_language, method="initialize") - async def client_listener(scope, message, languages, manager): + @lsp_message_listener("client", language_server=known_server, method="initialize") + async def client_listener(scope, message, language_server, manager): await handler_listened.put(message) @lsp_message_listener("client", method=r"not-a-method") async def other_client_listener( - scope, message, languages, manager + scope, message, language_server, manager ): # pragma: no cover await handler_listened.put(message) raise NotImplementedError("shouldn't get here") # some server listeners - @lsp_message_listener("server", language=None, method=None) - async def server_listener(scope, message, languages, manager): + @lsp_message_listener("server", language_server=None, method=None) + async def server_listener(scope, message, language_server, manager): await server_listened.put(message) - @lsp_message_listener("server", language=r"not-a-language") + @lsp_message_listener("server", language_server=r"not-a-language-server") async def other_server_listener( - scope, message, languages, manager + scope, message, language_server, manager ): # pragma: no cover await handler_listened.put(message) raise NotImplementedError("shouldn't get here") # an all listener @lsp_message_listener("all") - async def all_listener(scope, message, languages, manager): # pragma: no cover + async def all_listener( + scope, message, language_server, manager + ): # pragma: no cover await all_listened.put(message) assert len(manager._listeners["server"]) == 2 assert len(manager._listeners["client"]) == 2 assert len(manager._listeners["all"]) == 2 - ws_handler.open(known_language) + ws_handler.open(known_server) await ws_handler.on_message(jsonrpc_init_msg) diff --git a/py_src/jupyter_lsp/tests/test_session.py b/py_src/jupyter_lsp/tests/test_session.py index d96306e37..56d889899 100644 --- a/py_src/jupyter_lsp/tests/test_session.py +++ b/py_src/jupyter_lsp/tests/test_session.py @@ -5,7 +5,7 @@ from ..schema import SERVERS_RESPONSE -def assert_status_set(handler, expected_statuses, language=None): +def assert_status_set(handler, expected_statuses, language_server=None): handler.get() payload = handler._payload @@ -13,16 +13,16 @@ def assert_status_set(handler, expected_statuses, language=None): assert not errors statuses = { - s["status"] - for s in payload["sessions"] - if language is None or language in s["spec"]["languages"] + session["status"] + for session_server, session in payload["sessions"].items() + if language_server is None or language_server == session_server } - assert statuses == expected_statuses + assert statuses == expected_statuses, payload @pytest.mark.asyncio -async def test_start_known(known_language, handlers, jsonrpc_init_msg): - """ will a process start for a known language if a handler starts? +async def test_start_known(known_server, handlers, jsonrpc_init_msg): + """ will a process start for a known server if a handler starts? """ handler, ws_handler = handlers manager = handler.manager @@ -31,12 +31,11 @@ async def test_start_known(known_language, handlers, jsonrpc_init_msg): assert_status_set(handler, {"not_started"}) - ws_handler.open(known_language) - sessions = list(manager.sessions_for_handler(ws_handler)) - session = sessions[0] + ws_handler.open(known_server) + session = manager.sessions[ws_handler.language_server] assert session.process is not None - assert_status_set(handler, {"started"}, known_language) + assert_status_set(handler, {"started"}, known_server) await ws_handler.on_message(jsonrpc_init_msg) @@ -46,17 +45,16 @@ async def test_start_known(known_language, handlers, jsonrpc_init_msg): finally: ws_handler.on_close() - assert not list(manager.sessions_for_handler(ws_handler)) assert not session.handlers assert not session.process - assert_status_set(handler, {"stopped"}, known_language) + assert_status_set(handler, {"stopped"}, known_server) assert_status_set(handler, {"stopped", "not_started"}) @pytest.mark.asyncio -async def test_start_unknown(known_unknown_language, handlers, jsonrpc_init_msg): - """ will a process not start for an unknown if a handler starts? +async def test_start_unknown(known_unknown_server, handlers, jsonrpc_init_msg): + """ will a process not start for an unknown server if a handler starts? """ handler, ws_handler = handlers manager = handler.manager @@ -64,8 +62,7 @@ async def test_start_unknown(known_unknown_language, handlers, jsonrpc_init_msg) assert_status_set(handler, {"not_started"}) - ws_handler.open(known_unknown_language) - assert not list(manager.sessions_for_handler(ws_handler)) + ws_handler.open(known_unknown_server) assert_status_set(handler, {"not_started"}) @@ -73,5 +70,5 @@ async def test_start_unknown(known_unknown_language, handlers, jsonrpc_init_msg) assert_status_set(handler, {"not_started"}) ws_handler.on_close() - assert not list(manager.sessions_for_handler(ws_handler)) + assert not manager.sessions.get(ws_handler.language_server) assert_status_set(handler, {"not_started"}) diff --git a/py_src/jupyter_lsp/types.py b/py_src/jupyter_lsp/types.py index 0cabf451f..96026a57d 100644 --- a/py_src/jupyter_lsp/types.py +++ b/py_src/jupyter_lsp/types.py @@ -35,7 +35,7 @@ def __call__( self, scope: Text, message: LanguageServerMessage, - languages: List[Text], + language_server: Text, manager: "HasListeners", ) -> Awaitable[None]: ... @@ -66,31 +66,34 @@ class MessageListener(object): """ listener = None # type: HandlerListenerCallback - language = None # type: Optional[Pattern[Text]] + language_server = None # type: Optional[Pattern[Text]] method = None # type: Optional[Pattern[Text]] def __init__( self, listener: "HandlerListenerCallback", - language: Optional[Text], + language_server: Optional[Text], method: Optional[Text], ): self.listener = listener - self.language = re.compile(language) if language else None + self.language_server = re.compile(language_server) if language_server else None self.method = re.compile(method) if method else None async def __call__( self, scope: Text, message: LanguageServerMessage, - languages: List[Text], + language_server: Text, manager: "HasListeners", ) -> None: """ actually dispatch the message to the listener and capture any errors """ try: await self.listener( - scope=scope, message=message, languages=languages, manager=manager + scope=scope, + message=message, + language_server=language_server, + manager=manager, ) except Exception: # pragma: no cover manager.log.warn( @@ -100,7 +103,7 @@ async def __call__( exc_info=True, ) - def wants(self, message: LanguageServerMessage, languages: List[Text]): + def wants(self, message: LanguageServerMessage, language_server: Text): """ whether this listener wants a particular message `method` is currently the only message content discriminator, but not @@ -111,9 +114,9 @@ def wants(self, message: LanguageServerMessage, languages: List[Text]): if method is None or re.match(self.method, method) is None: return False - - return self.language is None or any( - [re.match(self.language, lang) is not None for lang in languages] + print("WANTS", self.language_server, language_server) + return self.language_server is None or re.match( + self.language_server, language_server ) def __repr__(self): @@ -121,7 +124,7 @@ def __repr__(self): "" + " language_server={self.language_server}>" ).format(self=self) @@ -134,7 +137,10 @@ class HasListeners: @classmethod def register_message_listener( - cls, scope: Text, language: Optional[Text] = None, method: Optional[Text] = None + cls, + scope: Text, + language_server: Optional[Text] = None, + method: Optional[Text] = None, ): """ register a listener for language server protocol messages """ @@ -142,7 +148,9 @@ def register_message_listener( def inner(listener: "HandlerListenerCallback") -> "HandlerListenerCallback": cls.unregister_message_listener(listener) cls._listeners[scope].append( - MessageListener(listener=listener, language=language, method=method) + MessageListener( + listener=listener, language_server=language_server, method=method + ) ) return listener @@ -160,7 +168,7 @@ def unregister_message_listener(cls, listener: "HandlerListenerCallback"): ] async def wait_for_listeners( - self, scope: MessageScope, message_str: Text, languages: List[Text] + self, scope: MessageScope, message_str: Text, language_server: Text ) -> None: scope_val = str(scope.value) listeners = self._listeners[scope_val] + self._listeners[MessageScope.ALL.value] @@ -169,9 +177,14 @@ async def wait_for_listeners( message = json.loads(message_str) futures = [ - listener(scope_val, message=message, languages=languages, manager=self) + listener( + scope_val, + message=message, + language_server=language_server, + manager=self, + ) for listener in listeners - if listener.wants(message, languages) + if listener.wants(message, language_server) ] if futures: diff --git a/py_src/jupyter_lsp/virtual_documents_shadow.py b/py_src/jupyter_lsp/virtual_documents_shadow.py index 8eaafd7ba..b0f507d81 100644 --- a/py_src/jupyter_lsp/virtual_documents_shadow.py +++ b/py_src/jupyter_lsp/virtual_documents_shadow.py @@ -101,7 +101,7 @@ def setup_shadow_filesystem(virtual_documents_uri): shadow_filesystem.mkdir(parents=True, exist_ok=True) @lsp_message_listener("client") - async def shadow_virtual_documents(scope, message, languages, manager): + async def shadow_virtual_documents(scope, message, language_server, manager): """Intercept a message with document contents creating a shadow file for it. Only create the shadow file if the URI matches the virtual documents URI. From a1e37cdeb453d27e8d4aa289560357618abcfa15 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 17 Feb 2020 00:29:04 -0500 Subject: [PATCH 02/17] start language server manager --- .../jupyterlab/components/statusbar.tsx | 27 +----- .../jupyterlab-lsp/src/connection_manager.ts | 48 +++++++++-- packages/jupyterlab-lsp/src/index.ts | 8 +- packages/jupyterlab-lsp/src/manager.ts | 86 +++++++++++++++++++ packages/jupyterlab-lsp/src/tokens.ts | 31 +++++++ py_src/jupyter_lsp/schema/schema.json | 23 ++--- py_src/jupyter_lsp/types.py | 1 - 7 files changed, 182 insertions(+), 42 deletions(-) create mode 100644 packages/jupyterlab-lsp/src/manager.ts create mode 100644 packages/jupyterlab-lsp/src/tokens.ts diff --git a/packages/jupyterlab-lsp/src/adapters/jupyterlab/components/statusbar.tsx b/packages/jupyterlab-lsp/src/adapters/jupyterlab/components/statusbar.tsx index d48aadfb3..1300eed80 100644 --- a/packages/jupyterlab-lsp/src/adapters/jupyterlab/components/statusbar.tsx +++ b/packages/jupyterlab-lsp/src/adapters/jupyterlab/components/statusbar.tsx @@ -21,8 +21,8 @@ import { DefaultIconReact } from '@jupyterlab/ui-components'; import { JupyterLabWidgetAdapter } from '../jl_adapter'; import { collect_documents, VirtualDocument } from '../../../virtual/document'; import { LSPConnection } from '../../../connection'; -import { PageConfig } from '@jupyterlab/coreutils'; import { DocumentConnectionManager } from '../../../connection_manager'; +import { ILanguageServerManager } from '../../../tokens'; interface IServerStatusProps { server: SCHEMA.LanguageServerSession; @@ -313,32 +313,11 @@ export namespace LSPStatus { */ export class Model extends VDomModel { server_extension_status: SCHEMA.ServersResponse = null; + language_server_manager: ILanguageServerManager; private _connection_manager: DocumentConnectionManager; - constructor() { - super(); - - // PathExt.join skips on of the slashes in https:// - let url = PageConfig.getBaseUrl() + 'lsp'; - fetch(url) - .then(response => { - // TODO: retry a few times - if (!response.ok) { - throw new Error(response.statusText); - } - response - .json() - .then( - (data: SCHEMA.ServersResponse) => - (this.server_extension_status = data) - ) - .catch(console.warn); - }) - .catch(console.error); - } - get available_servers(): Array { - return this.server_extension_status.sessions; + return Array.from(this.language_server_manager.sessions.values()); } get supported_languages(): Set { diff --git a/packages/jupyterlab-lsp/src/connection_manager.ts b/packages/jupyterlab-lsp/src/connection_manager.ts index 0e3000082..9abe594f8 100644 --- a/packages/jupyterlab-lsp/src/connection_manager.ts +++ b/packages/jupyterlab-lsp/src/connection_manager.ts @@ -7,6 +7,7 @@ import { sleep, until_ready } from './utils'; // Name-only import so as to not trigger inclusion in main bundle import * as ConnectionModuleType from './connection'; +import { TLanguageServerId, ILanguageServerManager } from './tokens'; export interface IDocumentConnectionData { virtual_document: VirtualDocument; @@ -49,9 +50,10 @@ export class DocumentConnectionManager { DocumentConnectionManager, Map >; + language_server_manager: ILanguageServerManager; private ignored_languages: Set; - constructor() { + constructor(options: DocumentConnectionManager.IOptions) { this.connections = new Map(); this.documents = new Map(); this.ignored_languages = new Set(); @@ -60,6 +62,8 @@ export class DocumentConnectionManager { this.disconnected = new Signal(this); this.closed = new Signal(this); this.documents_changed = new Signal(this); + this.language_server_manager = options.language_server_manager; + Private.setLanguageServerManager(options.language_server_manager); } connect_document_signals(virtual_document: VirtualDocument) { @@ -123,11 +127,16 @@ export class DocumentConnectionManager { language ); + const language_server_id = this.language_server_manager.getServerId({ + language + }); + // lazily load 1) the underlying library (1.5mb) and/or 2) a live WebSocket- // like connection: either already connected or potentiailly in the process // of connecting. const connection = await Private.connection( language, + language_server_id, uris, this.on_new_connection ); @@ -273,6 +282,10 @@ export class DocumentConnectionManager { } export namespace DocumentConnectionManager { + export interface IOptions { + language_server_manager: ILanguageServerManager; + } + export function solve_uris( virtual_document: VirtualDocument, language: string @@ -285,11 +298,20 @@ export namespace DocumentConnectionManager { ? rootUri : virtualDocumentsUri; + const language_server_id = Private.getLanguageServerManager().getServerId({ + language + }); + return { base: baseUri, document: URLExt.join(baseUri, virtual_document.uri), server: URLExt.join('ws://jupyter-lsp', language), - socket: URLExt.join(wsBase, 'lsp', language) + socket: URLExt.join( + wsBase, + ILanguageServerManager.URL_NS, + 'ws', + language_server_id + ) }; } @@ -305,14 +327,28 @@ export namespace DocumentConnectionManager { * Namespace primarily for language-keyed cache of LSPConnections */ namespace Private { - const _connections = new Map(); + const _connections: Map = new Map(); let _promise: Promise; + let _language_server_manager: ILanguageServerManager; + + export function getLanguageServerManager() { + return _language_server_manager; + } + export function setLanguageServerManager( + language_server_manager: ILanguageServerManager + ) { + if (_language_server_manager) { + return; + } + _language_server_manager = language_server_manager; + } /** * Return (or create and initialize) the WebSocket associated with the language */ export async function connection( language: string, + language_server_id: TLanguageServerId, uris: DocumentConnectionManager.IURIs, onCreate: (connection: LSPConnection) => void ): Promise { @@ -325,7 +361,7 @@ namespace Private { } const { LSPConnection } = await _promise; - let connection = _connections.get(language); + let connection = _connections.get(language_server_id); if (connection == null) { const socket = new WebSocket(uris.socket); @@ -336,12 +372,12 @@ namespace Private { }); // TODO: remove remaining unbounded users of connection.on connection.setMaxListeners(999); - _connections.set(language, connection); + _connections.set(language_server_id, connection); connection.connect(socket); onCreate(connection); } - connection = _connections.get(language); + connection = _connections.get(language_server_id); return connection; } diff --git a/packages/jupyterlab-lsp/src/index.ts b/packages/jupyterlab-lsp/src/index.ts index d7c9bf32a..7fba96f72 100644 --- a/packages/jupyterlab-lsp/src/index.ts +++ b/packages/jupyterlab-lsp/src/index.ts @@ -10,6 +10,8 @@ import { FileEditor, IEditorTracker } from '@jupyterlab/fileeditor'; import { ISettingRegistry } from '@jupyterlab/coreutils'; import { IDocumentManager } from '@jupyterlab/docmanager'; +import { LanguageServerManager } from './manager'; + import { FileEditorJumper } from '@krassowski/jupyterlab_go_to_definition/lib/jumpers/fileeditor'; import { NotebookJumper } from '@krassowski/jupyterlab_go_to_definition/lib/jumpers/notebook'; @@ -73,9 +75,13 @@ const plugin: JupyterFrontEndPlugin = { labShell: ILabShell, status_bar: IStatusBar ) => { - const connection_manager = new DocumentConnectionManager(); + const language_server_manager = new LanguageServerManager({}); + const connection_manager = new DocumentConnectionManager({ + language_server_manager + }); const status_bar_item = new LSPStatus(); + status_bar_item.model.language_server_manager = language_server_manager; status_bar_item.model.connection_manager = connection_manager; labShell.currentChanged.connect(() => { diff --git a/packages/jupyterlab-lsp/src/manager.ts b/packages/jupyterlab-lsp/src/manager.ts new file mode 100644 index 000000000..38136d7b8 --- /dev/null +++ b/packages/jupyterlab-lsp/src/manager.ts @@ -0,0 +1,86 @@ +import { Signal } from '@phosphor/signaling'; + +import { PageConfig, URLExt } from '@jupyterlab/coreutils'; +import { ServerConnection } from '@jupyterlab/services'; + +import { ILanguageServerManager, TSessionMap } from './tokens'; +import * as SCHEMA from './_schema'; + +export class LanguageServerManager implements ILanguageServerManager { + private _sessionsChanged: Signal = new Signal< + ILanguageServerManager, + void + >(this); + private _sessions: TSessionMap = new Map(); + private _settings: ServerConnection.ISettings; + private _baseUrl: string; + + constructor(options: ILanguageServerManager.IOptions) { + this._settings = options.settings || ServerConnection.makeSettings(); + this._baseUrl = options.baseUrl || PageConfig.getBaseUrl(); + this.fetchSessions().catch(console.warn); + } + + get statusUrl() { + return URLExt.join(this._baseUrl, ILanguageServerManager.URL_NS, 'status'); + } + + get sessionsChanged() { + return this._sessionsChanged; + } + + get sessions(): TSessionMap { + return this._sessions; + } + + getServerId(options: ILanguageServerManager.IGetServerIdOptions) { + // most things speak language + for (const [key, session] of this._sessions.entries()) { + if (options.language) { + if (session.spec.languages.indexOf(options.language) !== -1) { + return key; + } + } + } + return null; + } + + async fetchSessions() { + let response = await ServerConnection.makeRequest( + this.statusUrl, + { method: 'GET' }, + this._settings + ); + + if (!response.ok) { + throw new Error(response.statusText); + } + + let sessions: SCHEMA.Sessions; + + try { + sessions = (await response.json()).sessions; + } catch (err) { + console.warn(err); + return; + } + + for (const key of Object.keys(sessions)) { + if (this._sessions.has(key)) { + Object.assign(this._sessions.get(key), sessions[key]); + } else { + this._sessions.set(key, sessions[key]); + } + } + + const oldKeys = this._sessions.keys(); + + for (const oldKey in oldKeys) { + if (!sessions[oldKey]) { + this._sessions.delete(oldKey); + } + } + + this._sessionsChanged.emit(void 0); + } +} diff --git a/packages/jupyterlab-lsp/src/tokens.ts b/packages/jupyterlab-lsp/src/tokens.ts new file mode 100644 index 000000000..34fbb5ff2 --- /dev/null +++ b/packages/jupyterlab-lsp/src/tokens.ts @@ -0,0 +1,31 @@ +import { ISignal } from '@phosphor/signaling'; +import { ServerConnection } from '@jupyterlab/services'; + +import * as SCHEMA from './_schema'; + +export type TLanguageServerId = string; +export type TLanguageId = string; + +export type TSessionMap = Map; + +export interface ILanguageServerManager { + sessionsChanged: ISignal; + sessions: TSessionMap; + getServerId( + options: ILanguageServerManager.IGetServerIdOptions + ): TLanguageServerId; + fetchSessions(): Promise; + statusUrl: string; +} + +export namespace ILanguageServerManager { + export const URL_NS = 'lsp'; + export interface IOptions { + settings?: ServerConnection.ISettings; + baseUrl?: string; + } + export interface IGetServerIdOptions { + language?: TLanguageId; + mimeType?: string; + } +} diff --git a/py_src/jupyter_lsp/schema/schema.json b/py_src/jupyter_lsp/schema/schema.json index b238a82e1..67fd35b75 100644 --- a/py_src/jupyter_lsp/schema/schema.json +++ b/py_src/jupyter_lsp/schema/schema.json @@ -163,16 +163,7 @@ "servers-response": { "properties": { "sessions": { - "description": "named server sessions that are, could be, or were running", - "patternProperties": { - ".*": { - "$ref": "#/definitions/session" - } - }, - "additionalProperties": { - "$ref": "#/definitions/session" - }, - "type": "object" + "$ref": "#/definitions/sessions" }, "version": { "$ref": "#/definitions/current-version" @@ -181,6 +172,18 @@ "required": ["sessions", "version"], "type": "object" }, + "sessions": { + "description": "named server sessions that are, could be, or were running", + "patternProperties": { + ".*": { + "$ref": "#/definitions/session" + } + }, + "additionalProperties": { + "$ref": "#/definitions/session" + }, + "type": "object" + }, "session": { "additionalProperties": false, "description": "a language server session", diff --git a/py_src/jupyter_lsp/types.py b/py_src/jupyter_lsp/types.py index 96026a57d..b2f7e1957 100644 --- a/py_src/jupyter_lsp/types.py +++ b/py_src/jupyter_lsp/types.py @@ -114,7 +114,6 @@ def wants(self, message: LanguageServerMessage, language_server: Text): if method is None or re.match(self.method, method) is None: return False - print("WANTS", self.language_server, language_server) return self.language_server is None or re.match( self.language_server, language_server ) From b1acd079f3dfb7cbe9531b2c8d10dc8499429909 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 17 Feb 2020 00:44:09 -0500 Subject: [PATCH 03/17] mock language server manager for python --- .../src/adapters/codemirror/testutils.ts | 26 +++++++++++++++++++ .../jupyterlab-lsp/src/connection_manager.ts | 3 --- packages/jupyterlab-lsp/src/manager.ts | 4 +-- .../jupyterlab-lsp/src/virtual/editor.spec.ts | 12 +++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/packages/jupyterlab-lsp/src/adapters/codemirror/testutils.ts b/packages/jupyterlab-lsp/src/adapters/codemirror/testutils.ts index c9ebf9000..c1b91c0c0 100644 --- a/packages/jupyterlab-lsp/src/adapters/codemirror/testutils.ts +++ b/packages/jupyterlab-lsp/src/adapters/codemirror/testutils.ts @@ -22,6 +22,8 @@ import { ICellModel } from '@jupyterlab/cells'; import createNotebook = NBTestUtils.createNotebook; import { CodeMirrorAdapter } from './cm_adapter'; import { VirtualDocument } from '../../virtual/document'; +import { LanguageServerManager } from '../../manager'; +import { DocumentConnectionManager } from '../../connection_manager'; interface IFeatureTestEnvironment { host: HTMLElement; @@ -30,6 +32,18 @@ interface IFeatureTestEnvironment { dispose(): void; } +export class MockLanguageServerManager extends LanguageServerManager { + async fetchSessions() { + this._sessions = new Map(); + this._sessions.set('pyls', { + spec: { + languages: ['python'] + } + } as any); + this._sessionsChanged.emit(void 0); + } +} + export abstract class FeatureTestEnvironment implements IFeatureTestEnvironment { host: HTMLElement; @@ -129,6 +143,18 @@ export class FileEditorFeatureTestEnvironment extends FeatureTestEnvironment { host: this.host, model }); + + const LANGSERVER_MANAGER = new MockLanguageServerManager({}); + const CONNECTION_MANAGER = new DocumentConnectionManager({ + language_server_manager: LANGSERVER_MANAGER + }); + + const DEBUG = false; + + if (DEBUG) { + console.log(CONNECTION_MANAGER); + } + this.init(); } diff --git a/packages/jupyterlab-lsp/src/connection_manager.ts b/packages/jupyterlab-lsp/src/connection_manager.ts index 9abe594f8..d770851e5 100644 --- a/packages/jupyterlab-lsp/src/connection_manager.ts +++ b/packages/jupyterlab-lsp/src/connection_manager.ts @@ -337,9 +337,6 @@ namespace Private { export function setLanguageServerManager( language_server_manager: ILanguageServerManager ) { - if (_language_server_manager) { - return; - } _language_server_manager = language_server_manager; } diff --git a/packages/jupyterlab-lsp/src/manager.ts b/packages/jupyterlab-lsp/src/manager.ts index 38136d7b8..172497357 100644 --- a/packages/jupyterlab-lsp/src/manager.ts +++ b/packages/jupyterlab-lsp/src/manager.ts @@ -7,11 +7,11 @@ import { ILanguageServerManager, TSessionMap } from './tokens'; import * as SCHEMA from './_schema'; export class LanguageServerManager implements ILanguageServerManager { - private _sessionsChanged: Signal = new Signal< + protected _sessionsChanged: Signal = new Signal< ILanguageServerManager, void >(this); - private _sessions: TSessionMap = new Map(); + protected _sessions: TSessionMap = new Map(); private _settings: ServerConnection.ISettings; private _baseUrl: string; diff --git a/packages/jupyterlab-lsp/src/virtual/editor.spec.ts b/packages/jupyterlab-lsp/src/virtual/editor.spec.ts index a59f8c5cb..92a2ffea3 100644 --- a/packages/jupyterlab-lsp/src/virtual/editor.spec.ts +++ b/packages/jupyterlab-lsp/src/virtual/editor.spec.ts @@ -9,6 +9,7 @@ import { import * as CodeMirror from 'codemirror'; import { PageConfig } from '@jupyterlab/coreutils'; import { DocumentConnectionManager } from '../connection_manager'; +import { MockLanguageServerManager } from '../adapters/codemirror/testutils'; class VirtualEditorImplementation extends VirtualEditor { private cm_editor: CodeMirror.Editor; @@ -59,6 +60,17 @@ describe('VirtualEditor', () => { '/home/username/project/.virtual_documents' ); + const LANGSERVER_MANAGER = new MockLanguageServerManager({}); + const CONNECTION_MANAGER = new DocumentConnectionManager({ + language_server_manager: LANGSERVER_MANAGER + }); + + const DEBUG = false; + + if (DEBUG) { + console.log(CONNECTION_MANAGER); + } + let editor = new VirtualEditorImplementation( () => 'python', () => 'py', From f2f8ae5e1317f1d2240b4c7113d479d8c6d97cbd Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 17 Feb 2020 01:20:30 -0500 Subject: [PATCH 04/17] wuks docker --- atest/01_Editor.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atest/01_Editor.robot b/atest/01_Editor.robot index 75f47ac51..57c6d6d4b 100644 --- a/atest/01_Editor.robot +++ b/atest/01_Editor.robot @@ -19,7 +19,7 @@ CSS Docker ${def} = Set Variable xpath://span[contains(@class, 'cm-string')][contains(text(), 'PLANET')] - Editor Shows Features for Language Docker Dockerfile Diagnostics=Instruction has no arguments Jump to Definition=${def} Rename=${def} + Wait Until Keyword Succeeds 3x 100ms Editor Shows Features for Language Docker Dockerfile Diagnostics=Instruction has no arguments Jump to Definition=${def} Rename=${def} JS ${def} = Set Variable xpath:(//span[contains(@class, 'cm-variable')][contains(text(), 'fib')])[last()] From 61bf62d9692db49138d466773b6ac475877717af Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 17 Feb 2020 20:05:12 -0500 Subject: [PATCH 05/17] update changelog --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f11e27a68..9de7ad40d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - features - - opens a maximum of one WebSocket per language server ([#165][]) + - opens a maximum of one WebSocket per language server ([#165][], [#199][]) - lazy-loads language server protocol machinery ([#165][]) - waits much longer for slow-starting language servers ([#165][]) - cleans up documents, handlers, events, and signals more aggressively ([#165][]) @@ -22,6 +22,15 @@ [#165]: https://github.com/krassowski/jupyterlab-lsp/pull/165 +## `jupyter-lsp 0.8.0` (unreleased) + +- breaking changes + + - websockets are now serviced by implementation key, rather than language ([#199][]) + - introduces schema version `2`, reporting status by server ([#199][]) + +[#199]: https://github.com/krassowski/jupyterlab-lsp/pull/199 + ## `jupyter-lsp 0.7.0` - bugfixes From b0b35715a2c81e08efc84cb7be99dff668717f92 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Mon, 17 Feb 2020 20:06:22 -0500 Subject: [PATCH 06/17] more changelog --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9de7ad40d..910d7eb91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,8 +26,9 @@ - breaking changes - - websockets are now serviced by implementation key, rather than language ([#199][]) - - introduces schema version `2`, reporting status by server ([#199][]) + - websockets are now serviced by implementation key, rather than language + under `lsp/ws/` ([#199][]) + - introduces schema version `2`, reporting status by server at `lsp/status` ([#199][]) [#199]: https://github.com/krassowski/jupyterlab-lsp/pull/199 From 371eedf50f071d126105fe14d36747b5e54effbe Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 18 Feb 2020 07:27:53 -0500 Subject: [PATCH 07/17] move queue puts to main loop --- py_src/jupyter_lsp/session.py | 3 ++- py_src/jupyter_lsp/stdio.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/py_src/jupyter_lsp/session.py b/py_src/jupyter_lsp/session.py index 25e3ad3f3..625e42213 100644 --- a/py_src/jupyter_lsp/session.py +++ b/py_src/jupyter_lsp/session.py @@ -8,6 +8,7 @@ from copy import copy from datetime import datetime, timezone +from tornado.ioloop import IOLoop from tornado.queues import Queue from tornado.websocket import WebSocketHandler from traitlets import Bunch, Instance, Set, Unicode, UseEnum, observe @@ -130,7 +131,7 @@ def write(self, message): """ wrapper around the write queue to keep it mostly internal """ self.last_handler_message_at = self.now() - self.to_lsp.put_nowait(message) + IOLoop.current().add_callback(self.to_lsp.put_nowait, message) def now(self): return datetime.now(timezone.utc) diff --git a/py_src/jupyter_lsp/stdio.py b/py_src/jupyter_lsp/stdio.py index e032591e3..369d4d53f 100644 --- a/py_src/jupyter_lsp/stdio.py +++ b/py_src/jupyter_lsp/stdio.py @@ -15,6 +15,7 @@ from tornado.httputil import HTTPHeaders from tornado.queues import Queue +from tornado.ioloop import IOLoop from traitlets import Float, Instance, default from traitlets.config import LoggingConfigurable @@ -87,7 +88,7 @@ async def read(self) -> None: else: self.wake() - await self.queue.put(message) + IOLoop.current().add_callback(self.queue.put, message) except Exception: # pragma: no cover self.log.exception("%s couldn't enqueue message: %s", self, message) await self.sleep() From 35eaa1f9d0cdb0d6c4baea840dcbcb75019f47be Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 18 Feb 2020 07:47:27 -0500 Subject: [PATCH 08/17] do stdio read/write in threadpool --- atest/01_Editor.robot | 3 ++- py_src/jupyter_lsp/stdio.py | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/atest/01_Editor.robot b/atest/01_Editor.robot index 57c6d6d4b..a6d09f15a 100644 --- a/atest/01_Editor.robot +++ b/atest/01_Editor.robot @@ -19,7 +19,8 @@ CSS Docker ${def} = Set Variable xpath://span[contains(@class, 'cm-string')][contains(text(), 'PLANET')] - Wait Until Keyword Succeeds 3x 100ms Editor Shows Features for Language Docker Dockerfile Diagnostics=Instruction has no arguments Jump to Definition=${def} Rename=${def} + Wait Until Keyword Succeeds 3x 100ms Editor Shows Features for Language Docker Dockerfile Diagnostics=Instruction has no arguments + ... Jump to Definition=${def} Rename=${def} JS ${def} = Set Variable xpath:(//span[contains(@class, 'cm-variable')][contains(text(), 'fib')])[last()] diff --git a/py_src/jupyter_lsp/stdio.py b/py_src/jupyter_lsp/stdio.py index 369d4d53f..3704a9bff 100644 --- a/py_src/jupyter_lsp/stdio.py +++ b/py_src/jupyter_lsp/stdio.py @@ -11,11 +11,14 @@ import asyncio import io import os +from concurrent.futures import ThreadPoolExecutor from typing import Text +from tornado.concurrent import run_on_executor +from tornado.gen import convert_yielded from tornado.httputil import HTTPHeaders -from tornado.queues import Queue from tornado.ioloop import IOLoop +from tornado.queues import Queue from traitlets import Float, Instance, default from traitlets.config import LoggingConfigurable @@ -26,6 +29,8 @@ class LspStdIoBase(LoggingConfigurable): """ Non-blocking, queued base for communicating with stdio Language Servers """ + executor = None + stream = Instance(io.BufferedIOBase, help="the stream to read/write") queue = Instance(Queue, help="queue to get/put") @@ -35,6 +40,7 @@ def __repr__(self): # pragma: no cover def __init__(self, **kwargs): super().__init__(**kwargs) self.log.debug("%s initialized", self) + self.executor = ThreadPoolExecutor(max_workers=1) def close(self): self.stream.close() @@ -99,12 +105,12 @@ async def read_one(self) -> Text: message = "" headers = HTTPHeaders() - line = self._readline() + line = await convert_yielded(self._readline()) if line: while line and line.strip(): headers.parse_line(line) - line = self._readline() + line = await convert_yielded(self._readline()) content_length = int(headers.get("content-length", "0")) @@ -130,6 +136,7 @@ async def read_one(self) -> Text: return message + @run_on_executor def _readline(self) -> Text: """ Read a line (or immediately return None) """ @@ -143,7 +150,7 @@ class LspStdIoWriter(LspStdIoBase): """ Language Server stdio Writer """ - async def write(self): + async def write(self) -> None: """ Write to a Language Server until it closes """ while not self.stream.closed: @@ -151,9 +158,13 @@ async def write(self): try: body = message.encode("utf-8") response = "Content-Length: {}\r\n\r\n{}".format(len(body), message) - self.stream.write(response.encode("utf-8")) - self.stream.flush() + await convert_yielded(self._write_one(response.encode("utf-8"))) except Exception: # pragma: no cover self.log.exception("s couldn't write message: %s", self, response) finally: self.queue.task_done() + + @run_on_executor + def _write_one(self, message) -> None: + self.stream.write(message) + self.stream.flush() From 536790d135e55f2711a3f3a664e0e6b46d140f70 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Tue, 18 Feb 2020 07:59:08 -0500 Subject: [PATCH 09/17] use put_nowait for queue reading for consistency --- py_src/jupyter_lsp/stdio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py_src/jupyter_lsp/stdio.py b/py_src/jupyter_lsp/stdio.py index 3704a9bff..a31226685 100644 --- a/py_src/jupyter_lsp/stdio.py +++ b/py_src/jupyter_lsp/stdio.py @@ -94,7 +94,7 @@ async def read(self) -> None: else: self.wake() - IOLoop.current().add_callback(self.queue.put, message) + IOLoop.current().add_callback(self.queue.put_nowait, message) except Exception: # pragma: no cover self.log.exception("%s couldn't enqueue message: %s", self, message) await self.sleep() From 99dc206c58de8edbb05b7e8c2ef36953f4d26e2d Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 22 Feb 2020 14:22:00 -0500 Subject: [PATCH 10/17] promote connection/language server managers to test environment --- .../src/adapters/codemirror/testutils.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/jupyterlab-lsp/src/adapters/codemirror/testutils.ts b/packages/jupyterlab-lsp/src/adapters/codemirror/testutils.ts index c1b91c0c0..24d7ee45a 100644 --- a/packages/jupyterlab-lsp/src/adapters/codemirror/testutils.ts +++ b/packages/jupyterlab-lsp/src/adapters/codemirror/testutils.ts @@ -129,6 +129,8 @@ export abstract class FeatureTestEnvironment export class FileEditorFeatureTestEnvironment extends FeatureTestEnvironment { ce_editor: CodeMirrorEditor; + connection_manager: DocumentConnectionManager; + language_server_manager: LanguageServerManager; constructor( language = () => 'python', @@ -144,17 +146,11 @@ export class FileEditorFeatureTestEnvironment extends FeatureTestEnvironment { model }); - const LANGSERVER_MANAGER = new MockLanguageServerManager({}); - const CONNECTION_MANAGER = new DocumentConnectionManager({ - language_server_manager: LANGSERVER_MANAGER + this.language_server_manager = new MockLanguageServerManager({}); + this.connection_manager = new DocumentConnectionManager({ + language_server_manager: this.language_server_manager }); - const DEBUG = false; - - if (DEBUG) { - console.log(CONNECTION_MANAGER); - } - this.init(); } From 45da44945b80af223963c2b5d19249a17766944a Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 22 Feb 2020 14:25:40 -0500 Subject: [PATCH 11/17] add listener api changes to docs notebook --- docs/Extending.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/Extending.ipynb b/docs/Extending.ipynb index 4fc7b1fbe..38215f055 100644 --- a/docs/Extending.ipynb +++ b/docs/Extending.ipynb @@ -120,9 +120,9 @@ "def load_jupyter_server_extension(nbapp):\n", "\n", " @lsp_message_listener(\"all\")\n", - " async def my_listener(scope, message, languages, manager):\n", - " print(\"received a {} {} message about {}\".format(\n", - " scope, message[\"method\"], languages\n", + " async def my_listener(scope, message, language_server, manager):\n", + " print(\"received a {} {} message from {}\".format(\n", + " scope, message[\"method\"], language_server\n", " ))\n", "```\n", "\n", @@ -138,7 +138,7 @@ "Fine-grained controls are available as part of the Python API. Pass these as\n", "named arguments to `lsp_message_listener`.\n", "\n", - "- `languages`: a regular expression of languages\n", + "- `language_server`: a regular expression of language servers\n", "- `method`: a regular expression of LSP JSON-RPC method names" ] } From 6078327aa300524e2fef0cfd4a95702607cd19e8 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 22 Feb 2020 14:55:35 -0500 Subject: [PATCH 12/17] some updating of the architecture docs --- docs/Architecture.ipynb | 598 ++++++++++++++++++++++++++++++++++++- docs/dot/as-is/backend.dot | 36 ++- 2 files changed, 617 insertions(+), 17 deletions(-) diff --git a/docs/Architecture.ipynb b/docs/Architecture.ipynb index d8110fc93..3270f6813 100644 --- a/docs/Architecture.ipynb +++ b/docs/Architecture.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "hide_input": true }, @@ -19,11 +19,14 @@ "\n", "import graphviz\n", "import IPython\n", + "import re\n", "\n", "\n", "@IPython.core.magic.register_line_cell_magic\n", "def dot(line, cell=None):\n", - " return graphviz.Source(pathlib.Path(line).read_text() if line else cell)" + " src = graphviz.Source(pathlib.Path(line).read_text() if line else cell)._repr_svg_()\n", + " src = re.sub(r'\n", + "\n", + "backend\n", + "\n", + "Front End\n", + "\n", + "cluster_io\n", + "\n", + "IO\n", + "\n", + "\n", + "cluster_server\n", + "\n", + "Server\n", + "\n", + "\n", + "cluster_lab\n", + "\n", + "JupyterLab\n", + "\n", + "\n", + "cluster_lang\n", + "\n", + "Per Language\n", + "\n", + "\n", + "cluster_doc\n", + "\n", + "Per Document\n", + "\n", + "\n", + "cluster_doc_lsp\n", + "\n", + "LSP Client\n", + "\n", + "\n", + "cluster_features\n", + "\n", + "Features\n", + "\n", + "\n", + "\n", + "Keyboard\n", + "The Keyboard\n", + "\n", + "\n", + "\n", + "CodeMirrors\n", + "CodeMirrors\n", + "\n", + "\n", + "\n", + "Keyboard--CodeMirrors\n", + "\n", + "\n", + "\n", + "\n", + "Mouse\n", + "The Mouse\n", + "\n", + "\n", + "\n", + "Mouse--CodeMirrors\n", + "\n", + "\n", + "\n", + "\n", + "RestAPI\n", + "The REST API\n", + "\n", + "\n", + "\n", + "WebSocketHandlers\n", + "WebSocketHandlers\n", + "\n", + "\n", + "\n", + "Plugin\n", + "Plugin\n", + "\n", + "\n", + "\n", + "StatusBar\n", + "StatusBar\n", + "\n", + "\n", + "\n", + "Plugin--StatusBar\n", + "\n", + "\n", + "\n", + "\n", + "ConnectionManager\n", + "ConnectionManager\n", + "\n", + "\n", + "\n", + "Plugin--ConnectionManager\n", + "\n", + "\n", + "\n", + "\n", + "Widget\n", + "Widget\n", + "\n", + "\n", + "\n", + "Plugin--Widget\n", + "\n", + "\n", + "\n", + "\n", + "StatusBar--RestAPI\n", + "\n", + "\n", + "\n", + "\n", + "Connection\n", + "Connection\n", + "\n", + "\n", + "\n", + "StatusBar--Connection\n", + "\n", + "\n", + "\n", + "\n", + "WidgetTrackers\n", + "WidgetTrackers\n", + "\n", + "\n", + "\n", + "WidgetTrackers--Plugin\n", + "\n", + "\n", + "\n", + "\n", + "WidgetTrackers--Widget\n", + "\n", + "\n", + "\n", + "\n", + "DiagnosticPanel\n", + "DiagnosticPanel\n", + "\n", + "\n", + "\n", + "Connection--ConnectionManager\n", + "\n", + "\n", + "\n", + "\n", + "MessageConnection\n", + "MessageConnection\n", + "\n", + "\n", + "\n", + "Connection--MessageConnection\n", + "\n", + "\n", + "\n", + "\n", + "Feature\n", + "\n", + "\n", + "\n", + "Connection--Feature\n", + "\n", + "\n", + "\n", + "\n", + "WebSocket\n", + "WebSocket\n", + "\n", + "\n", + "\n", + "MessageConnection--WebSocket\n", + "\n", + "\n", + "\n", + "\n", + "WebSocket--WebSocketHandlers\n", + "\n", + "\n", + "\n", + "\n", + "Widget--CodeMirrors\n", + "\n", + "\n", + "\n", + "\n", + "WidgetAdapter\n", + "WidgetAdapter\n", + "\n", + "\n", + "\n", + "Widget--WidgetAdapter\n", + "\n", + "\n", + "\n", + "\n", + "CodeMirrorAdapters\n", + "CodeMirrorAdapters\n", + "\n", + "\n", + "\n", + "CodeMirrors--CodeMirrorAdapters\n", + "\n", + "\n", + "\n", + "\n", + "CodeMirrors--Feature\n", + "\n", + "\n", + "\n", + "\n", + "WidgetAdapter--Plugin\n", + "\n", + "\n", + "\n", + "\n", + "VirtualEditor\n", + "VirtualEditor\n", + "\n", + "\n", + "\n", + "WidgetAdapter--VirtualEditor\n", + "\n", + "\n", + "\n", + "\n", + "VirtualDocument\n", + "VirtualDocument\n", + "\n", + "\n", + "\n", + "VirtualEditor--VirtualDocument\n", + "\n", + "\n", + "\n", + "\n", + "VirtualEditor--Feature\n", + "\n", + "\n", + "\n", + "\n", + "VirtualDocument--Feature\n", + "\n", + "\n", + "\n", + "\n", + "CodeMirrorAdapters--WidgetAdapter\n", + "\n", + "\n", + "\n", + "\n", + "Completion\n", + "Completion\n", + "\n", + "\n", + "\n", + "Diagnostics\n", + "Diagnostics\n", + "\n", + "\n", + "\n", + "Diagnostics--DiagnosticPanel\n", + "\n", + "\n", + "\n", + "\n", + "Highlight\n", + "Highlight\n", + "\n", + "\n", + "\n", + "Hover\n", + "Hover\n", + "\n", + "\n", + "\n", + "Jump\n", + "Jump\n", + "\n", + "\n", + "\n", + "Rename\n", + "Rename\n", + "\n", + "\n", + "\n", + "Signature\n", + "Signature\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "%dot dot/as-is/frontend.dot" ] @@ -62,11 +374,285 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": { "hide_input": true }, - "outputs": [], + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "backend\n", + "\n", + "Back End\n", + "\n", + "cluster_browser\n", + "\n", + "Browser\n", + "\n", + "\n", + "cluster_notebook\n", + "\n", + "Jupyter Server\n", + "\n", + "\n", + "cluster_lsp\n", + "\n", + "LSP ServerExtension\n", + "\n", + "\n", + "cluster_lsp\n", + "\n", + "Per Language Server\n", + "\n", + "\n", + "cluster_read_thread\n", + "\n", + "Reader Thread\n", + "\n", + "\n", + "cluster_write_thread\n", + "\n", + "Writer Thread\n", + "\n", + "\n", + "cluster_ls_process\n", + "\n", + "Subprocess\n", + "\n", + "\n", + "cluster_lab_ext\n", + "\n", + "JupyterLab ServerExtension\n", + "\n", + "\n", + "cluster_files\n", + "\n", + "Files\n", + "\n", + "\n", + "cluster_config\n", + "\n", + "Config\n", + "\n", + "\n", + "cluster_jupyter_cfg\n", + "\n", + "Jupyter Config Paths\n", + "\n", + "\n", + "cluster_python_pkgs\n", + "\n", + "Python Packages\n", + "\n", + "\n", + "\n", + "Clients\n", + "Clients\n", + "\n", + "\n", + "\n", + "WebSockets\n", + "WebSockets\n", + "\n", + "\n", + "\n", + "Clients--WebSockets\n", + "\n", + "\n", + "\n", + "\n", + "RestAPI\n", + "The REST API\n", + "\n", + "\n", + "\n", + "Clients--RestAPI\n", + "\n", + "\n", + "\n", + "\n", + "PageConfig\n", + "The Page Config\n", + "\n", + "\n", + "\n", + "Clients--PageConfig\n", + "\n", + "\n", + "\n", + "\n", + "Manager\n", + "The\n", + "Language\n", + "Server\n", + "Manager\n", + "\n", + "\n", + "\n", + "SpecFinders\n", + "SpecFinders\n", + "\n", + "\n", + "\n", + "Manager--SpecFinders\n", + "\n", + "\n", + "\n", + "\n", + "Spec\n", + "Spec\n", + "\n", + "\n", + "\n", + "Manager--Spec\n", + "\n", + "\n", + "\n", + "\n", + "Session\n", + "Session\n", + "\n", + "\n", + "\n", + "Manager--Session\n", + "\n", + "\n", + "\n", + "\n", + "Handlers\n", + "Handlers\n", + "\n", + "\n", + "\n", + "Handlers--Manager\n", + "\n", + "\n", + "\n", + "\n", + "WebSockets--Handlers\n", + "\n", + "\n", + "\n", + "\n", + "Spec--Session\n", + "\n", + "\n", + "\n", + "\n", + "Reader\n", + "Reader\n", + "\n", + "\n", + "\n", + "Session--Reader\n", + "\n", + "\n", + "\n", + "\n", + "Writer\n", + "Writer\n", + "\n", + "\n", + "\n", + "Session--Writer\n", + "\n", + "\n", + "\n", + "\n", + "Listeners\n", + "Listeners\n", + "\n", + "\n", + "\n", + "Session--Listeners\n", + "\n", + "\n", + "\n", + "\n", + "LanguageServer\n", + "Language\n", + "Server\n", + "\n", + "\n", + "\n", + "Reader--LanguageServer\n", + "\n", + "\n", + "\n", + "\n", + "Writer--LanguageServer\n", + "\n", + "\n", + "\n", + "\n", + "RealFiles\n", + "Real Files\n", + "\n", + "\n", + "\n", + "LanguageServer--RealFiles\n", + "\n", + "\n", + "\n", + "\n", + "VirtualFiles\n", + "Virtual Files\n", + "\n", + "\n", + "\n", + "LanguageServer--VirtualFiles\n", + "\n", + "\n", + "\n", + "\n", + "Listeners--VirtualFiles\n", + "\n", + "\n", + "\n", + "\n", + "RestAPI--Manager\n", + "\n", + "\n", + "\n", + "\n", + "PageConfig--Manager\n", + "\n", + "\n", + "\n", + "\n", + "Traitlets\n", + "Traitlets\n", + "\n", + "\n", + "\n", + "Traitlets--SpecFinders\n", + "\n", + "\n", + "\n", + "\n", + "EntryPoints\n", + "EntryPoints\n", + "\n", + "\n", + "\n", + "EntryPoints--SpecFinders\n", + "\n", + "\n", + "\n", + "" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "%dot dot/as-is/backend.dot" ] diff --git a/docs/dot/as-is/backend.dot b/docs/dot/as-is/backend.dot index f4119282a..f17530a71 100644 --- a/docs/dot/as-is/backend.dot +++ b/docs/dot/as-is/backend.dot @@ -4,19 +4,32 @@ graph backend { fontname="sans-serif" node[shape=none fontname="sans-serif"] edge[fontname="sans-serif"] - subgraph cluster_browser { graph[fontcolor=grey color=grey] + + subgraph cluster_browser { label=Browser Clients } + subgraph cluster_notebook { label="Jupyter Server" subgraph cluster_lsp { label="LSP ServerExtension" - Sessions Manager[label="The\nLanguage\nServer\nManager"] - Handlers + Handlers WebSockets SpecFinders - Specs + subgraph cluster_lsp { label="Per Language Server" + Spec + Session + subgraph cluster_read_thread { label="Reader Thread" + Reader + } + subgraph cluster_write_thread { label="Writer Thread" + Writer + } + subgraph cluster_ls_process { label="Subprocess" + LanguageServer[label="Language\nServer"] + } + } Listeners RestAPI[label="The REST API"] } @@ -37,12 +50,13 @@ graph backend { } } } - - LanguageServers[label="Language\nServers"] + Clients -- {RestAPI PageConfig} -- Manager - Clients -- WebSockets -- Handlers -- Sessions -- LanguageServers - {Manager Traitlets EntryPoints} -- SpecFinders -- Specs - Specs -- Sessions - Sessions -- Listeners -- VirtualFiles - LanguageServers -- {RealFiles VirtualFiles} + Clients -- WebSockets -- Handlers -- Manager + Session -- {Reader Writer} -- LanguageServer + {Manager Traitlets EntryPoints} -- SpecFinders + Manager -- {Spec Session} + Spec -- Session + Session -- Listeners -- VirtualFiles + LanguageServer -- {RealFiles VirtualFiles} } From 5bfe3a28b8176d944e0f13737d6bdaffad3fff63 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 22 Feb 2020 15:02:32 -0500 Subject: [PATCH 13/17] revisit frontend arch a touch --- docs/Architecture.ipynb | 593 +----------------------------------- docs/dot/as-is/frontend.dot | 21 +- 2 files changed, 17 insertions(+), 597 deletions(-) diff --git a/docs/Architecture.ipynb b/docs/Architecture.ipynb index 3270f6813..84017361e 100644 --- a/docs/Architecture.ipynb +++ b/docs/Architecture.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "hide_input": true }, @@ -47,320 +47,11 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "hide_input": true }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "backend\n", - "\n", - "Front End\n", - "\n", - "cluster_io\n", - "\n", - "IO\n", - "\n", - "\n", - "cluster_server\n", - "\n", - "Server\n", - "\n", - "\n", - "cluster_lab\n", - "\n", - "JupyterLab\n", - "\n", - "\n", - "cluster_lang\n", - "\n", - "Per Language\n", - "\n", - "\n", - "cluster_doc\n", - "\n", - "Per Document\n", - "\n", - "\n", - "cluster_doc_lsp\n", - "\n", - "LSP Client\n", - "\n", - "\n", - "cluster_features\n", - "\n", - "Features\n", - "\n", - "\n", - "\n", - "Keyboard\n", - "The Keyboard\n", - "\n", - "\n", - "\n", - "CodeMirrors\n", - "CodeMirrors\n", - "\n", - "\n", - "\n", - "Keyboard--CodeMirrors\n", - "\n", - "\n", - "\n", - "\n", - "Mouse\n", - "The Mouse\n", - "\n", - "\n", - "\n", - "Mouse--CodeMirrors\n", - "\n", - "\n", - "\n", - "\n", - "RestAPI\n", - "The REST API\n", - "\n", - "\n", - "\n", - "WebSocketHandlers\n", - "WebSocketHandlers\n", - "\n", - "\n", - "\n", - "Plugin\n", - "Plugin\n", - "\n", - "\n", - "\n", - "StatusBar\n", - "StatusBar\n", - "\n", - "\n", - "\n", - "Plugin--StatusBar\n", - "\n", - "\n", - "\n", - "\n", - "ConnectionManager\n", - "ConnectionManager\n", - "\n", - "\n", - "\n", - "Plugin--ConnectionManager\n", - "\n", - "\n", - "\n", - "\n", - "Widget\n", - "Widget\n", - "\n", - "\n", - "\n", - "Plugin--Widget\n", - "\n", - "\n", - "\n", - "\n", - "StatusBar--RestAPI\n", - "\n", - "\n", - "\n", - "\n", - "Connection\n", - "Connection\n", - "\n", - "\n", - "\n", - "StatusBar--Connection\n", - "\n", - "\n", - "\n", - "\n", - "WidgetTrackers\n", - "WidgetTrackers\n", - "\n", - "\n", - "\n", - "WidgetTrackers--Plugin\n", - "\n", - "\n", - "\n", - "\n", - "WidgetTrackers--Widget\n", - "\n", - "\n", - "\n", - "\n", - "DiagnosticPanel\n", - "DiagnosticPanel\n", - "\n", - "\n", - "\n", - "Connection--ConnectionManager\n", - "\n", - "\n", - "\n", - "\n", - "MessageConnection\n", - "MessageConnection\n", - "\n", - "\n", - "\n", - "Connection--MessageConnection\n", - "\n", - "\n", - "\n", - "\n", - "Feature\n", - "\n", - "\n", - "\n", - "Connection--Feature\n", - "\n", - "\n", - "\n", - "\n", - "WebSocket\n", - "WebSocket\n", - "\n", - "\n", - "\n", - "MessageConnection--WebSocket\n", - "\n", - "\n", - "\n", - "\n", - "WebSocket--WebSocketHandlers\n", - "\n", - "\n", - "\n", - "\n", - "Widget--CodeMirrors\n", - "\n", - "\n", - "\n", - "\n", - "WidgetAdapter\n", - "WidgetAdapter\n", - "\n", - "\n", - "\n", - "Widget--WidgetAdapter\n", - "\n", - "\n", - "\n", - "\n", - "CodeMirrorAdapters\n", - "CodeMirrorAdapters\n", - "\n", - "\n", - "\n", - "CodeMirrors--CodeMirrorAdapters\n", - "\n", - "\n", - "\n", - "\n", - "CodeMirrors--Feature\n", - "\n", - "\n", - "\n", - "\n", - "WidgetAdapter--Plugin\n", - "\n", - "\n", - "\n", - "\n", - "VirtualEditor\n", - "VirtualEditor\n", - "\n", - "\n", - "\n", - "WidgetAdapter--VirtualEditor\n", - "\n", - "\n", - "\n", - "\n", - "VirtualDocument\n", - "VirtualDocument\n", - "\n", - "\n", - "\n", - "VirtualEditor--VirtualDocument\n", - "\n", - "\n", - "\n", - "\n", - "VirtualEditor--Feature\n", - "\n", - "\n", - "\n", - "\n", - "VirtualDocument--Feature\n", - "\n", - "\n", - "\n", - "\n", - "CodeMirrorAdapters--WidgetAdapter\n", - "\n", - "\n", - "\n", - "\n", - "Completion\n", - "Completion\n", - "\n", - "\n", - "\n", - "Diagnostics\n", - "Diagnostics\n", - "\n", - "\n", - "\n", - "Diagnostics--DiagnosticPanel\n", - "\n", - "\n", - "\n", - "\n", - "Highlight\n", - "Highlight\n", - "\n", - "\n", - "\n", - "Hover\n", - "Hover\n", - "\n", - "\n", - "\n", - "Jump\n", - "Jump\n", - "\n", - "\n", - "\n", - "Rename\n", - "Rename\n", - "\n", - "\n", - "\n", - "Signature\n", - "Signature\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "%dot dot/as-is/frontend.dot" ] @@ -374,285 +65,11 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": { "hide_input": true }, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "\n", - "backend\n", - "\n", - "Back End\n", - "\n", - "cluster_browser\n", - "\n", - "Browser\n", - "\n", - "\n", - "cluster_notebook\n", - "\n", - "Jupyter Server\n", - "\n", - "\n", - "cluster_lsp\n", - "\n", - "LSP ServerExtension\n", - "\n", - "\n", - "cluster_lsp\n", - "\n", - "Per Language Server\n", - "\n", - "\n", - "cluster_read_thread\n", - "\n", - "Reader Thread\n", - "\n", - "\n", - "cluster_write_thread\n", - "\n", - "Writer Thread\n", - "\n", - "\n", - "cluster_ls_process\n", - "\n", - "Subprocess\n", - "\n", - "\n", - "cluster_lab_ext\n", - "\n", - "JupyterLab ServerExtension\n", - "\n", - "\n", - "cluster_files\n", - "\n", - "Files\n", - "\n", - "\n", - "cluster_config\n", - "\n", - "Config\n", - "\n", - "\n", - "cluster_jupyter_cfg\n", - "\n", - "Jupyter Config Paths\n", - "\n", - "\n", - "cluster_python_pkgs\n", - "\n", - "Python Packages\n", - "\n", - "\n", - "\n", - "Clients\n", - "Clients\n", - "\n", - "\n", - "\n", - "WebSockets\n", - "WebSockets\n", - "\n", - "\n", - "\n", - "Clients--WebSockets\n", - "\n", - "\n", - "\n", - "\n", - "RestAPI\n", - "The REST API\n", - "\n", - "\n", - "\n", - "Clients--RestAPI\n", - "\n", - "\n", - "\n", - "\n", - "PageConfig\n", - "The Page Config\n", - "\n", - "\n", - "\n", - "Clients--PageConfig\n", - "\n", - "\n", - "\n", - "\n", - "Manager\n", - "The\n", - "Language\n", - "Server\n", - "Manager\n", - "\n", - "\n", - "\n", - "SpecFinders\n", - "SpecFinders\n", - "\n", - "\n", - "\n", - "Manager--SpecFinders\n", - "\n", - "\n", - "\n", - "\n", - "Spec\n", - "Spec\n", - "\n", - "\n", - "\n", - "Manager--Spec\n", - "\n", - "\n", - "\n", - "\n", - "Session\n", - "Session\n", - "\n", - "\n", - "\n", - "Manager--Session\n", - "\n", - "\n", - "\n", - "\n", - "Handlers\n", - "Handlers\n", - "\n", - "\n", - "\n", - "Handlers--Manager\n", - "\n", - "\n", - "\n", - "\n", - "WebSockets--Handlers\n", - "\n", - "\n", - "\n", - "\n", - "Spec--Session\n", - "\n", - "\n", - "\n", - "\n", - "Reader\n", - "Reader\n", - "\n", - "\n", - "\n", - "Session--Reader\n", - "\n", - "\n", - "\n", - "\n", - "Writer\n", - "Writer\n", - "\n", - "\n", - "\n", - "Session--Writer\n", - "\n", - "\n", - "\n", - "\n", - "Listeners\n", - "Listeners\n", - "\n", - "\n", - "\n", - "Session--Listeners\n", - "\n", - "\n", - "\n", - "\n", - "LanguageServer\n", - "Language\n", - "Server\n", - "\n", - "\n", - "\n", - "Reader--LanguageServer\n", - "\n", - "\n", - "\n", - "\n", - "Writer--LanguageServer\n", - "\n", - "\n", - "\n", - "\n", - "RealFiles\n", - "Real Files\n", - "\n", - "\n", - "\n", - "LanguageServer--RealFiles\n", - "\n", - "\n", - "\n", - "\n", - "VirtualFiles\n", - "Virtual Files\n", - "\n", - "\n", - "\n", - "LanguageServer--VirtualFiles\n", - "\n", - "\n", - "\n", - "\n", - "Listeners--VirtualFiles\n", - "\n", - "\n", - "\n", - "\n", - "RestAPI--Manager\n", - "\n", - "\n", - "\n", - "\n", - "PageConfig--Manager\n", - "\n", - "\n", - "\n", - "\n", - "Traitlets\n", - "Traitlets\n", - "\n", - "\n", - "\n", - "Traitlets--SpecFinders\n", - "\n", - "\n", - "\n", - "\n", - "EntryPoints\n", - "EntryPoints\n", - "\n", - "\n", - "\n", - "EntryPoints--SpecFinders\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "execution_count": 34, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "%dot dot/as-is/backend.dot" ] diff --git a/docs/dot/as-is/frontend.dot b/docs/dot/as-is/frontend.dot index ab0b86ee1..4c48f711f 100644 --- a/docs/dot/as-is/frontend.dot +++ b/docs/dot/as-is/frontend.dot @@ -16,12 +16,15 @@ graph backend { WebSocketHandlers } subgraph cluster_lab { label="JupyterLab" - Plugin - StatusBar + subgraph cluster_singletons { label="Singletons" + Plugin + StatusBarItem + DiagnosticPanel + ConnectionManager + LanguageServerManager + } WidgetTrackers - DiagnosticPanel - ConnectionManager - + subgraph cluster_lang {label="Per Language" Connection MessageConnection @@ -51,14 +54,14 @@ graph backend { {Connection CodeMirrors VirtualDocument VirtualEditor} -- Feature[lhead=cluster_features]; Diagnostics -- DiagnosticPanel - Connection -- ConnectionManager + Connection -- ConnectionManager -- LanguageServerManager -- RestAPI CodeMirrors -- CodeMirrorAdapters -- WidgetAdapter WidgetAdapter -- VirtualEditor -- VirtualDocument WidgetTrackers -- {Plugin -- Widget} - StatusBar -- Connection - Widget -- WidgetAdapter -- Plugin -- StatusBar -- RestAPI + StatusBarItem -- Connection + Widget -- WidgetAdapter -- Plugin -- StatusBarItem -- LanguageServerManager Widget -- CodeMirrors - Plugin -- ConnectionManager + Plugin -- {ConnectionManager LanguageServerManager} Connection -- MessageConnection -- WebSocket -- WebSocketHandlers {Keyboard Mouse} -- CodeMirrors } \ No newline at end of file From 61e633d867500724747238a0154254edcb243ed2 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 22 Feb 2020 15:03:01 -0500 Subject: [PATCH 14/17] ignore graphviz mypy errors --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index 1918f207e..0ffa7d965 100644 --- a/setup.cfg +++ b/setup.cfg @@ -128,3 +128,6 @@ ignore_missing_imports = True [mypy-nbsphinx] ignore_missing_imports = True + +[mypy-graphviz] +ignore_missing_imports = True From 5bc7b5fa800efa45451c9e5bbfeda0d1f4744419 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 22 Feb 2020 15:19:45 -0500 Subject: [PATCH 15/17] also make shadow file ops async --- .../tests/test_virtual_documents_shadow.py | 40 +++++++++++++------ .../jupyter_lsp/virtual_documents_shadow.py | 36 ++++++++++++----- 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/py_src/jupyter_lsp/tests/test_virtual_documents_shadow.py b/py_src/jupyter_lsp/tests/test_virtual_documents_shadow.py index 1977dcdd5..31f69ccb2 100644 --- a/py_src/jupyter_lsp/tests/test_virtual_documents_shadow.py +++ b/py_src/jupyter_lsp/tests/test_virtual_documents_shadow.py @@ -11,32 +11,46 @@ ) -def test_read(tmp_path): +@pytest.mark.asyncio +async def test_read(tmp_path): path = tmp_path / "existing.py" path.write_text("a\ntest") - file = EditableFile(path) - assert file.lines == ["a", "test"] + editable_file = EditableFile(path) + + await editable_file.read() + + assert editable_file.lines == ["a", "test"] + + +@pytest.mark.asyncio +async def test_read_missing(tmp_path): path = tmp_path / "missing.py" - file = EditableFile(path) - assert file.lines == [""] + missing_file = EditableFile(path) + + await missing_file.read() + + assert missing_file.lines == [""] -def test_apply_change(tmp_path): +@pytest.mark.asyncio +async def test_apply_change(tmp_path): # inserting text path = tmp_path / "test.py" - file = EditableFile(path) - file.apply_change("new\ntext", **file.full_range) - assert file.lines == ["new", "text"] + editable_file = EditableFile(path) + await editable_file.read() + + editable_file.apply_change("new\ntext", **editable_file.full_range) + assert editable_file.lines == ["new", "text"] # modifying a range - file.apply_change( + editable_file.apply_change( "ves", start={"line": 1, "character": 0}, end={"line": 1, "character": 3} ) - assert file.lines == ["new", "vest"] + assert editable_file.lines == ["new", "vest"] - file.apply_change("", **file.full_range) - assert file.lines == [""] + editable_file.apply_change("", **editable_file.full_range) + assert editable_file.lines == [""] def test_extract_or_none(): diff --git a/py_src/jupyter_lsp/virtual_documents_shadow.py b/py_src/jupyter_lsp/virtual_documents_shadow.py index b0f507d81..e0bd8a752 100644 --- a/py_src/jupyter_lsp/virtual_documents_shadow.py +++ b/py_src/jupyter_lsp/virtual_documents_shadow.py @@ -1,10 +1,17 @@ # flake8: noqa: W503 +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from shutil import rmtree +from tornado.concurrent import run_on_executor +from tornado.gen import convert_yielded + from .manager import lsp_message_listener from .paths import file_uri_to_path +# TODO: make configurable +MAX_WORKERS = 4 + def extract_or_none(obj, path): for crumb in path: @@ -16,11 +23,19 @@ def extract_or_none(obj, path): class EditableFile: + executor = ThreadPoolExecutor(max_workers=MAX_WORKERS) + def __init__(self, path): # Python 3.5 relict: self.path = Path(path) if isinstance(path, str) else path - self.lines = self.read_lines() + async def read(self): + self.lines = await convert_yielded(self.read_lines()) + + async def write(self): + return await convert_yielded(self.write_lines()) + + @run_on_executor def read_lines(self): # empty string required by the assumptions of the gluing algorithm lines = [""] @@ -30,6 +45,11 @@ def read_lines(self): pass return lines + @run_on_executor + def write_lines(self): + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text("\n".join(self.lines)) + @staticmethod def trim(lines: list, character: int, side: int): needs_glue = False @@ -63,10 +83,6 @@ def apply_change(self, text: str, start, end): + after[1 if needs_glue_right else None :] ) or [""] - def write(self): - self.path.parent.mkdir(parents=True, exist_ok=True) - self.path.write_text("\n".join(self.lines)) - @property def full_range(self): start = {"line": 0, "character": 0} @@ -125,7 +141,9 @@ async def shadow_virtual_documents(scope, message, language_server, manager): return path = file_uri_to_path(uri) - file = EditableFile(path) + editable_file = EditableFile(path) + + await editable_file.read() text = extract_or_none(document, ["text"]) @@ -148,10 +166,10 @@ async def shadow_virtual_documents(scope, message, language_server, manager): ) for change in changes[:1]: - change_range = change.get("range", file.full_range) - file.apply_change(change["text"], **change_range) + change_range = change.get("range", editable_file.full_range) + editable_file.apply_change(change["text"], **change_range) - file.write() + await editable_file.write() return path From aaf623a4de8c98c16115139252015a83b8e76bf3 Mon Sep 17 00:00:00 2001 From: Nicholas Bollweg Date: Sat, 22 Feb 2020 15:19:53 -0500 Subject: [PATCH 16/17] lint architecture --- docs/Architecture.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Architecture.ipynb b/docs/Architecture.ipynb index 84017361e..1860ed104 100644 --- a/docs/Architecture.ipynb +++ b/docs/Architecture.ipynb @@ -16,16 +16,16 @@ "outputs": [], "source": [ "import pathlib\n", + "import re\n", "\n", "import graphviz\n", "import IPython\n", - "import re\n", "\n", "\n", "@IPython.core.magic.register_line_cell_magic\n", "def dot(line, cell=None):\n", " src = graphviz.Source(pathlib.Path(line).read_text() if line else cell)._repr_svg_()\n", - " src = re.sub(r' Date: Sat, 22 Feb 2020 15:52:43 -0500 Subject: [PATCH 17/17] add note about threading concerns --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a5fa79f2..f018390d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ under `lsp/ws/` ([#199][]) - introduces schema version `2`, reporting status by server at `lsp/status` ([#199][]) +- bugfixes: + - handles language server reading/writing and shadow file operations in threads ([#199][]) + [#199]: https://github.com/krassowski/jupyterlab-lsp/pull/199 ### `jupyter-lsp 0.7.0`