Make code extractors and replacements modular

packages/jupyterlab-lsp/src/adapters/adapter.ts

@@ -124,6 +124,14 @@ export abstract class WidgetAdapter<T extends IDocumentWidget> {
     this.widget.disposed.connect(this.dispose, this);
   }
 
+  protected get foreign_code_extractors() {
+    return this.extension.foreign_code_extractors;
+  }
+
+  protected get code_overrides() {
+    return this.extension.code_overrides;
+  }
+
   on_connection_closed(
     manager: DocumentConnectionManager,
     { virtual_document }: IDocumentConnectionData

packages/jupyterlab-lsp/src/adapters/notebook/notebook.ts

@@ -1,8 +1,6 @@
 import { WidgetAdapter } from '../adapter';
 import { Notebook, NotebookPanel } from '@jupyterlab/notebook';
 import { until_ready } from '../../utils';
-import { language_specific_overrides } from '../../magics/defaults';
-import { foreign_code_extractors } from '../../extractors/defaults';
 import { Cell } from '@jupyterlab/cells';
 import * as nbformat from '@jupyterlab/nbformat';
 import ILanguageInfoMetadata = nbformat.ILanguageInfoMetadata;
@@ -161,8 +159,8 @@ export class NotebookAdapter extends WidgetAdapter<NotebookPanel> {
     return new VirtualDocument({
       language: this.language,
       path: this.document_path,
-      overrides_registry: language_specific_overrides,
-      foreign_code_extractors: foreign_code_extractors,
+      overrides_registry: this.code_overrides,
+      foreign_code_extractors: this.foreign_code_extractors,
       file_extension: this.language_file_extension,
       // notebooks are continuous, each cell is dependent on the previous one
       standalone: false,

packages/jupyterlab-lsp/src/editor_integration/feature.spec.ts

@@ -9,8 +9,8 @@ import {
 } from './testutils';
 import * as lsProtocol from 'vscode-languageserver-protocol';
 import * as nbformat from '@jupyterlab/nbformat';
-import { language_specific_overrides } from '../magics/defaults';
-import { foreign_code_extractors } from '../extractors/defaults';
+import { overrides } from '../transclusions/ipython/overrides';
+import { foreign_code_extractors } from '../transclusions/ipython/extractors';
 import { NotebookModel } from '@jupyterlab/notebook';
 import { PageConfig } from '@jupyterlab/coreutils';
 import { CodeMirrorIntegration } from './codemirror';
@@ -183,7 +183,12 @@ describe('Feature', () => {
 
       beforeEach(() => {
         environment = new NotebookFeatureTestEnvironment({
-          overrides_registry: language_specific_overrides,
+          overrides_registry: {
+            python: {
+              cell: overrides.filter(override => override.scope == 'cell'),
+              line: overrides.filter(override => override.scope == 'line')
+            }
+          },
           foreign_code_extractors: foreign_code_extractors
         });
 

packages/jupyterlab-lsp/src/editor_integration/testutils.ts

@@ -44,6 +44,8 @@ import { ServiceManager } from '@jupyterlab/services';
 import { FeatureManager, ILSPExtension } from '../index';
 import { JupyterFrontEnd } from '@jupyterlab/application';
 import { IFeatureSettings } from '../feature';
+import { IForeignCodeExtractorsRegistry } from '../extractors/types';
+import { ICodeOverridesRegistry } from '../overrides/tokens';
 
 export interface ITestEnvironment {
   document_options: VirtualDocument.IOptions;
@@ -88,6 +90,8 @@ export class MockExtension implements ILSPExtension {
   language_server_manager: LanguageServerManager;
   feature_manager: ILSPFeatureManager;
   editor_type_manager: ILSPVirtualEditorManager;
+  foreign_code_extractors: IForeignCodeExtractorsRegistry;
+  code_overrides: ICodeOverridesRegistry;
 
   constructor() {
     this.app = null;
@@ -102,6 +106,8 @@ export class MockExtension implements ILSPExtension {
       name: 'CodeMirrorEditor',
       supports: CodeMirrorEditor
     });
+    this.foreign_code_extractors = {};
+    this.code_overrides = {};
   }
 }
 

packages/jupyterlab-lsp/src/extractors/manager.ts

@@ -0,0 +1,28 @@
+import { JupyterFrontEndPlugin } from '@jupyterlab/application';
+import { ILSPCodeExtractorsManager } from '../tokens';
+import { IForeignCodeExtractor, IForeignCodeExtractorsRegistry } from './types';
+
+export class CodeExtractorsManager implements ILSPCodeExtractorsManager {
+  registry: IForeignCodeExtractorsRegistry;
+
+  constructor() {
+    this.registry = {};
+  }
+
+  register(extractor: IForeignCodeExtractor, host_language: string): void {
+    if (!this.registry.hasOwnProperty(host_language)) {
+      this.registry[host_language] = [];
+    }
+    this.registry[host_language].push(extractor);
+  }
+}
+
+export const CODE_EXTRACTORS_MANAGER: JupyterFrontEndPlugin<ILSPCodeExtractorsManager> = {
+  id: ILSPCodeExtractorsManager.name,
+  requires: [],
+  activate: app => {
+    return new CodeExtractorsManager();
+  },
+  provides: ILSPCodeExtractorsManager,
+  autoStart: true
+};

packages/jupyterlab-lsp/src/extractors/regexp.ts

@@ -1,6 +1,6 @@
 import { IExtractedCode, IForeignCodeExtractor } from './types';
 import { position_at_offset } from '../positioning';
-import { replacer } from '../magics/overrides';
+import { replacer } from '../overrides/tokens';
 import { CodeEditor } from '@jupyterlab/codeeditor';
 
 export class RegExpForeignCodeExtractor implements IForeignCodeExtractor {

packages/jupyterlab-lsp/src/extractors/testutils.ts

@@ -0,0 +1,32 @@
+import { CodeEditor } from '@jupyterlab/codeeditor';
+import { IVirtualDocumentBlock, VirtualDocument } from '../virtual/document';
+import { expect } from 'chai';
+
+export function extract_code(document: VirtualDocument, code: string) {
+  return document.extract_foreign_code(code, null, {
+    line: 0,
+    column: 0
+  });
+}
+
+export function get_the_only_pair(
+  foreign_document_map: Map<CodeEditor.IRange, IVirtualDocumentBlock>
+) {
+  expect(foreign_document_map.size).to.equal(1);
+
+  let range = foreign_document_map.keys().next().value;
+  let { virtual_document } = foreign_document_map.get(range);
+
+  return { range, virtual_document };
+}
+
+export function get_the_only_virtual(
+  foreign_document_map: Map<CodeEditor.IRange, IVirtualDocumentBlock>
+) {
+  let { virtual_document } = get_the_only_pair(foreign_document_map);
+  return virtual_document;
+}
+
+export function wrap_in_python_lines(line: string) {
+  return 'print("some code before")\n' + line + '\n' + 'print("and after")\n';
+}

packages/jupyterlab-lsp/src/extractors/types.ts

@@ -40,7 +40,7 @@ export interface IExtractedCode {
  */
 export interface IForeignCodeExtractor {
   /**
-   * The foreign language.
+   * The foreign language. TODO what is language, what is host language, ILanguageInfo
    */
   language: string;
 

packages/jupyterlab-lsp/src/features/diagnostics/diagnostics.spec.ts

@@ -15,8 +15,7 @@ import {
 } from '../../editor_integration/testutils';
 import { CodeMirrorEditor } from '@jupyterlab/codemirror';
 import { is_equal } from '../../positioning';
-import { language_specific_overrides } from '../../magics/defaults';
-import { foreign_code_extractors } from '../../extractors/defaults';
+import { foreign_code_extractors } from '../../transclusions/ipython/extractors';
 import * as lsProtocol from 'vscode-languageserver-protocol';
 import { CodeDiagnostics as LSPDiagnosticsSettings } from '../../_diagnostics';
 
@@ -144,7 +143,7 @@ describe('Diagnostics', () => {
 
     beforeEach(() => {
       env = new NotebookFeatureTestEnvironment({
-        overrides_registry: language_specific_overrides,
+        overrides_registry: {},
         foreign_code_extractors
       });
       feature = env.init_integration({

packages/jupyterlab-lsp/src/index.ts

@@ -14,6 +14,7 @@ import { LSPStatus } from './components/statusbar';
 import { DocumentConnectionManager } from './connection_manager';
 import {
   ILSPAdapterManager,
+  ILSPCodeExtractorsManager,
   ILSPFeatureManager,
   ILSPVirtualEditorManager,
   PLUGIN_ID,
@@ -35,6 +36,13 @@ import { LabIcon } from '@jupyterlab/ui-components';
 import codeCheckSvg from '../style/icons/code-check.svg';
 import { DIAGNOSTICS_PLUGIN } from './features/diagnostics';
 import { COMPLETION_PLUGIN } from './features/completion';
+import { CODE_EXTRACTORS_MANAGER } from './extractors/manager';
+import { IForeignCodeExtractorsRegistry } from './extractors/types';
+import {
+  ILSPCodeOverridesManager,
+  ICodeOverridesRegistry
+} from './overrides/tokens';
+import { DEFAULT_TRANSCLUSIONS } from './transclusions/defaults';
 
 export const codeCheckIcon = new LabIcon({
   name: 'lsp:codeCheck',
@@ -98,6 +106,8 @@ export interface ILSPExtension {
   language_server_manager: LanguageServerManager;
   feature_manager: ILSPFeatureManager;
   editor_type_manager: ILSPVirtualEditorManager;
+  foreign_code_extractors: IForeignCodeExtractorsRegistry;
+  code_overrides: ICodeOverridesRegistry;
 }
 
 export class LSPExtension implements ILSPExtension {
@@ -113,7 +123,9 @@ export class LSPExtension implements ILSPExtension {
     paths: IPaths,
     status_bar: IStatusBar,
     adapterManager: ILSPAdapterManager,
-    public editor_type_manager: ILSPVirtualEditorManager
+    public editor_type_manager: ILSPVirtualEditorManager,
+    private code_extractors_manager: ILSPCodeExtractorsManager,
+    private code_overrides_manager: ILSPCodeOverridesManager
   ) {
     this.language_server_manager = new LanguageServerManager({});
     this.connection_manager = new DocumentConnectionManager({
@@ -165,6 +177,14 @@ export class LSPExtension implements ILSPExtension {
     });
   }
 
+  get foreign_code_extractors() {
+    return this.code_extractors_manager.registry;
+  }
+
+  get code_overrides() {
+    return this.code_overrides_manager.registry;
+  }
+
   private updateOptions(settings: ISettingRegistry.ISettings) {
     const options = settings.composite;
 
@@ -186,7 +206,9 @@ const plugin: JupyterFrontEndPlugin<ILSPFeatureManager> = {
     IPaths,
     IStatusBar,
     ILSPAdapterManager,
-    ILSPVirtualEditorManager
+    ILSPVirtualEditorManager,
+    ILSPCodeExtractorsManager,
+    ILSPCodeOverridesManager
   ],
   activate: (app, ...args) => {
     let extension = new LSPExtension(
@@ -198,7 +220,9 @@ const plugin: JupyterFrontEndPlugin<ILSPFeatureManager> = {
         IPaths,
         IStatusBar,
         ILSPAdapterManager,
-        ILSPVirtualEditorManager
+        ILSPVirtualEditorManager,
+        ILSPCodeExtractorsManager,
+        ILSPCodeOverridesManager
       ])
     );
     return extension.feature_manager;
@@ -218,12 +242,14 @@ const default_features: JupyterFrontEndPlugin<void>[] = [
 ];
 
 const plugins: JupyterFrontEndPlugin<any>[] = [
+  CODE_EXTRACTORS_MANAGER,
   WIDGET_ADAPTER_MANAGER,
   NOTEBOOK_ADAPTER,
   FILE_EDITOR_ADAPTER,
   VIRTUAL_EDITOR_MANAGER,
   CODEMIRROR_VIRTUAL_EDITOR,
   plugin,
+  ...DEFAULT_TRANSCLUSIONS,
   ...default_features
 ];