Merge pull request #553 from julioyildo/bigquery-magic

Add BigQuery magic (%%bigquery)

CHANGELOG.md

@@ -1,6 +1,10 @@
 ## CHANGELOG
 
-### `@krassowski/jupyterlab-lsp 3.4.2` (unreleased)
+### `@krassowski/jupyterlab-lsp 3.5.0` (unreleased)
+
+- features:
+
+  - adds `%%bigquery` IPython cell magic support for BigQuery ([#553], thanks @julioyildo)
 
 - bug fixes:
 
@@ -9,6 +13,7 @@
 
 [#544]: https://github.com/krassowski/jupyterlab-lsp/pull/544
 [#547]: https://github.com/krassowski/jupyterlab-lsp/pull/547
+[#553]: https://github.com/krassowski/jupyterlab-lsp/pull/553
 
 ### `jupyter-lsp 1.1.4` (2020-02-21)
 

packages/jupyterlab-lsp/src/transclusions/defaults.ts

@@ -1,10 +1,12 @@
 import { JupyterFrontEndPlugin } from '@jupyterlab/application';
 import { IPYTHON_RPY2_TRANSCLUSIONS } from './ipython-rpy2';
 import { IPYTHON_SQL_TRANSCLUSIONS } from './ipython-sql';
+import { IPYTHON_BIGQUERY_TRANSCLUSIONS } from './ipython-bigquery';
 import { IPYTHON_TRANSCLUSIONS } from './ipython';
 
 export const DEFAULT_TRANSCLUSIONS: JupyterFrontEndPlugin<void>[] = [
   IPYTHON_RPY2_TRANSCLUSIONS,
   IPYTHON_SQL_TRANSCLUSIONS,
+  IPYTHON_BIGQUERY_TRANSCLUSIONS,
   IPYTHON_TRANSCLUSIONS
 ];

packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts

@@ -0,0 +1,42 @@
+import { VirtualDocument } from '../../virtual/document';
+import { expect } from 'chai';
+import { foreign_code_extractors } from './extractors';
+import { extract_code, get_the_only_virtual } from '../../extractors/testutils';
+
+describe('Bigquery SQL extractors', () => {
+  let document: VirtualDocument;
+
+  function extract(code: string) {
+    return extract_code(document, code);
+  }
+
+  beforeEach(() => {
+    document = new VirtualDocument({
+      language: 'python',
+      path: 'test.ipynb',
+      overrides_registry: {},
+      foreign_code_extractors: foreign_code_extractors,
+      standalone: false,
+      file_extension: 'py',
+      has_lsp_supported_file: false
+    });
+  });
+
+  afterEach(() => {
+    document.clear();
+  });
+
+  describe('%%bigquery cell magic', () => {
+    it('extracts simple commands', () => {
+      let code = "%%bigquery\nselect * from character\nwhere abbrev = 'ALICE'";
+      let { cell_code_kept, foreign_document_map } = extract(code);
+
+      expect(cell_code_kept).to.equal(code);
+      let document = get_the_only_virtual(foreign_document_map);
+      expect(document.language).to.equal('sql');
+      expect(document.value).to.equal(
+        "select * from character\nwhere abbrev = 'ALICE'\n"
+      );
+    });
+  });
+});

packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts

@@ -0,0 +1,35 @@
+import { IForeignCodeExtractorsRegistry } from '../../extractors/types';
+import { RegExpForeignCodeExtractor } from '../../extractors/regexp';
+
+export const SQL_URL_PATTERN = '(?:(?:.*?)://(?:.*))';
+// note: -a/--connection_arguments and -f/--file are not supported yet
+const single_argument_options = [
+  '--destination_table',
+  '--project',
+  '--use_bqstorage_api',
+  '--use_rest_api',
+  '--use_legacy_sql',
+  '--verbose',
+  '--params'
+];
+const zero_argument_options = ['-l', '--connections'];
+
+const COMMAND_PATTERN =
+  '(?:' +
+  (zero_argument_options.join('|') +
+    '|' +
+    single_argument_options.map(command => command + ' \\w+').join('|')) +
+  ')';
+
+export let foreign_code_extractors: IForeignCodeExtractorsRegistry = {
+  // general note: to match new lines use [^] instead of dot, unless the target is ES2018, then use /s
+  python: [
+    new RegExpForeignCodeExtractor({
+      language: 'sql',
+      pattern: `^%%bigquery(?: (?:${SQL_URL_PATTERN}|${COMMAND_PATTERN}|(?:\\w+ << )|(?:\\w+@\\w+)))?\n?(.+\n)?([^]*)`,
+      extract_to_foreign: '$1$2',
+      is_standalone: true,
+      file_extension: 'sql'
+    })
+  ]
+};

packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/index.ts

@@ -0,0 +1,20 @@
+import { JupyterFrontEndPlugin } from '@jupyterlab/application';
+import { ILSPCodeExtractorsManager, PLUGIN_ID } from '../../tokens';
+import { foreign_code_extractors } from './extractors';
+
+/**
+ * Implements extraction of code for IPython Magics for BigQuery, see:
+ * https://googleapis.dev/python/bigquery/latest/magics.html.
+ */
+export const IPYTHON_BIGQUERY_TRANSCLUSIONS: JupyterFrontEndPlugin<void> = {
+  id: PLUGIN_ID + ':ipython-bigquery',
+  requires: [ILSPCodeExtractorsManager],
+  activate: (app, extractors_manager: ILSPCodeExtractorsManager) => {
+    for (let language of Object.keys(foreign_code_extractors)) {
+      for (let extractor of foreign_code_extractors[language]) {
+        extractors_manager.register(extractor, language);
+      }
+    }
+  },
+  autoStart: true
+};