From 2117e1ecf5c58ff0f96b44c70ed52f1ffa4a5344 Mon Sep 17 00:00:00 2001 From: Serhat YILDIRIM Date: Tue, 9 Mar 2021 09:36:34 +0000 Subject: [PATCH 1/8] Add ipython-bigquery magic --- .../ipython-bigquery/extractors.spec.ts | 136 ++++++++++++++++++ .../ipython-bigquery/extractors.ts | 45 ++++++ .../transclusions/ipython-bigquery/index.ts | 20 +++ 3 files changed, 201 insertions(+) create mode 100644 packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts create mode 100644 packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts create mode 100644 packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/index.ts diff --git a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts new file mode 100644 index 000000000..839a4a2c8 --- /dev/null +++ b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts @@ -0,0 +1,136 @@ +import { VirtualDocument } from '../../virtual/document'; +import { expect } from 'chai'; +import { foreign_code_extractors, SQL_URL_PATTERN } from './extractors'; +import { + extract_code, + get_the_only_virtual, + wrap_in_python_lines +} from '../../extractors/testutils'; + +describe('Bigquery SQL extractors', () => { + let document: VirtualDocument; + + function extract(code: string) { + return extract_code(document, code); + } + + beforeEach(() => { + document = new VirtualDocument({ + language: 'python', + path: 'test.ipynb', + overrides_registry: {}, + foreign_code_extractors: foreign_code_extractors, + standalone: false, + file_extension: 'py', + has_lsp_supported_file: false + }); + }); + + afterEach(() => { + document.clear(); + }); + + /* + describe('SQL url pattern', () => { + it('matches connection strings', () => { + const correct_urls = [ + 'mysql+pymysql://scott:tiger@localhost/foo', + 'oracle://scott:tiger@127.0.0.1:1521/sidname', + 'sqlite://', + 'sqlite:///foo.db', + 'mssql+pyodbc://username:password@host/database?driver=SQL+Server+Native+Client+11.0', + 'impala://hserverhost:port/default?kerberos_service_name=hive&auth_mechanism=GSSAPI' + ]; + const pattern = new RegExp(SQL_URL_PATTERN); + for (let url of correct_urls) { + expect(pattern.test(url)).to.equal(true); + } + }); + }); +*/ + describe('%bigquery line magic', () => { + it('extracts simple commands', () => { + let code = wrap_in_python_lines('%bigquery select * from work'); + let { cell_code_kept, foreign_document_map } = extract(code); + + // should not be removed, but left for the static analysis (using magic overrides) + expect(cell_code_kept).to.equal(code); + let document = get_the_only_virtual(foreign_document_map); + expect(document.language).to.equal('sql'); + expect(document.value).to.equal('select * from work\n'); + }); +/* + it('leaves out the connection specification', () => { + let code = wrap_in_python_lines( + '%sql postgresql://will:longliveliz@localhost/shakes' + ); + let foreign_document_map = extract(code).foreign_document_map; + let document = get_the_only_virtual(foreign_document_map); + expect(document.language).to.equal('sql'); + expect(document.value).to.equal('\n'); + }); + + it('leaves out options', () => { + let code = wrap_in_python_lines('%bigquery -l'); + let foreign_document_map = extract(code).foreign_document_map; + let document = get_the_only_virtual(foreign_document_map); + expect(document.language).to.equal('sql'); + expect(document.value).to.equal('\n'); + }); + }); +*/ + describe('%%bigquery cell magic', () => { + it('extracts simple commands', () => { + let code = "%%bigquery\nselect * from character\nwhere abbrev = 'ALICE'"; + let { cell_code_kept, foreign_document_map } = extract(code); + + expect(cell_code_kept).to.equal(code); + let document = get_the_only_virtual(foreign_document_map); + expect(document.language).to.equal('sql'); + expect(document.value).to.equal( + "select * from character\nwhere abbrev = 'ALICE'\n" + ); + }); +/* + it('leaves out the connection specification', () => { + let code = + "%%sql postgresql://will:longliveliz@localhost/shakes\nselect * from character\nwhere abbrev = 'ALICE'"; + let { foreign_document_map } = extract(code); + + let document = get_the_only_virtual(foreign_document_map); + expect(document.language).to.equal('sql'); + expect(document.value).to.equal( + "select * from character\nwhere abbrev = 'ALICE'\n" + ); + }); + + + it('leaves out the variable assignment', () => { + let code = '%%sql works << SELECT title, year\nFROM work'; + let { foreign_document_map } = extract(code); + + let document = get_the_only_virtual(foreign_document_map); + expect(document.language).to.equal('sql'); + expect(document.value).to.equal('SELECT title, year\nFROM work\n'); + }); + + it('leaves out existing connection references', () => { + let code = '%%sql will@shakes\nSELECT title, year\nFROM work'; + let { foreign_document_map } = extract(code); + + let document = get_the_only_virtual(foreign_document_map); + expect(document.language).to.equal('sql'); + expect(document.value).to.equal('SELECT title, year\nFROM work\n'); + }); +*/ +/* it('leaves out persist option', () => { + let code = '%%bigquery --persist dataframe\nSELECT * FROM dataframe;'; + let { foreign_document_map } = extract(code); + let document = get_the_only_virtual(foreign_document_map); + expect(document.language).to.equal('sql'); + expect(document.value).to.equal('SELECT * FROM dataframe;\n'); + }); +*/ + + }); +}); diff --git a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts new file mode 100644 index 000000000..c7e2d564c --- /dev/null +++ b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts @@ -0,0 +1,45 @@ +import { IForeignCodeExtractorsRegistry } from '../../extractors/types'; +import { RegExpForeignCodeExtractor } from '../../extractors/regexp'; + +export const SQL_URL_PATTERN = '(?:(?:.*?)://(?:.*))'; +// note: -a/--connection_arguments and -f/--file are not supported yet +const single_argument_options = [ + '--destination_table', + '--project', + '--use_bqstorage_api', + '--use_rest_api', + '--use_legacy_sql', + '--verbose', + '--params' +]; +const zero_argument_options = ['-l', '--connections']; + +const COMMAND_PATTERN = + '(?:' + + (zero_argument_options.join('|') + + '|' + + single_argument_options.map(command => command + ' \\w+').join('|')) + + ')'; + +export let foreign_code_extractors: IForeignCodeExtractorsRegistry = { + // general note: to match new lines use [^] instead of dot, unless the target is ES2018, then use /s + python: [ + new RegExpForeignCodeExtractor({ + language: 'sql', + pattern: `^%%bigquery(?: (?:${SQL_URL_PATTERN}|${COMMAND_PATTERN}|(?:\\w+ << )|(?:\\w+@\\w+)))?\n?(.+\n)?([^]*)`, + extract_to_foreign: '$1$2', + is_standalone: true, + file_extension: 'sql' + }) +/* + , + new RegExpForeignCodeExtractor({ + language: 'sql', + pattern: `(?:^|\n)%bigquery (?:${SQL_URL_PATTERN}|${COMMAND_PATTERN}|(.*))\n?`, + extract_to_foreign: '$1', + is_standalone: false, + file_extension: 'sql' + }) + */ + ] +}; diff --git a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/index.ts b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/index.ts new file mode 100644 index 000000000..e4e72997d --- /dev/null +++ b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/index.ts @@ -0,0 +1,20 @@ +import { JupyterFrontEndPlugin } from '@jupyterlab/application'; +import { ILSPCodeExtractorsManager, PLUGIN_ID } from '../../tokens'; +import { foreign_code_extractors } from './extractors'; + +/** + * Implements extraction of code for IPython Magics for BigQuery, see: + * https://googleapis.dev/python/bigquery/latest/magics.html. + */ +export const IPYTHON_BIGQUERY_TRANSCLUSIONS: JupyterFrontEndPlugin = { + id: PLUGIN_ID + ':ipython-bigquery', + requires: [ILSPCodeExtractorsManager], + activate: (app, extractors_manager: ILSPCodeExtractorsManager) => { + for (let language of Object.keys(foreign_code_extractors)) { + for (let extractor of foreign_code_extractors[language]) { + extractors_manager.register(extractor, language); + } + } + }, + autoStart: true +}; From 7ecb9bd57afa66d9ef0e6dae20f7814f8036801f Mon Sep 17 00:00:00 2001 From: Serhat YILDIRIM Date: Tue, 9 Mar 2021 09:57:33 +0000 Subject: [PATCH 2/8] Add ipython-bigquery to defaults.ts --- packages/jupyterlab-lsp/src/transclusions/defaults.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/jupyterlab-lsp/src/transclusions/defaults.ts b/packages/jupyterlab-lsp/src/transclusions/defaults.ts index 6cd3398c6..dbbc9f298 100644 --- a/packages/jupyterlab-lsp/src/transclusions/defaults.ts +++ b/packages/jupyterlab-lsp/src/transclusions/defaults.ts @@ -1,10 +1,12 @@ import { JupyterFrontEndPlugin } from '@jupyterlab/application'; import { IPYTHON_RPY2_TRANSCLUSIONS } from './ipython-rpy2'; import { IPYTHON_SQL_TRANSCLUSIONS } from './ipython-sql'; +import { IPYTHON_BIGQUERY_TRANSCLUSIONS } from './ipython-bigquery'; import { IPYTHON_TRANSCLUSIONS } from './ipython'; export const DEFAULT_TRANSCLUSIONS: JupyterFrontEndPlugin[] = [ IPYTHON_RPY2_TRANSCLUSIONS, IPYTHON_SQL_TRANSCLUSIONS, - IPYTHON_TRANSCLUSIONS + IPYTHON_TRANSCLUSIONS, + IPYTHON_BIGQUERY_TRANSCLUSIONS ]; From f5e8e1a88ec12d590ea9e5311bd2eb2770062cac Mon Sep 17 00:00:00 2001 From: Serhat YILDIRIM Date: Tue, 9 Mar 2021 09:59:18 +0000 Subject: [PATCH 3/8] update ipython-bigquery spec : comment %bigquery line magic --- .../src/transclusions/ipython-bigquery/extractors.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts index 839a4a2c8..6b403cbfb 100644 --- a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts +++ b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts @@ -48,7 +48,7 @@ describe('Bigquery SQL extractors', () => { }); }); */ - describe('%bigquery line magic', () => { +/* describe('%bigquery line magic', () => { it('extracts simple commands', () => { let code = wrap_in_python_lines('%bigquery select * from work'); let { cell_code_kept, foreign_document_map } = extract(code); @@ -58,7 +58,7 @@ describe('Bigquery SQL extractors', () => { let document = get_the_only_virtual(foreign_document_map); expect(document.language).to.equal('sql'); expect(document.value).to.equal('select * from work\n'); - }); + });*/ /* it('leaves out the connection specification', () => { let code = wrap_in_python_lines( From b00b6184f1b3505b1ef5eb0e770a12594cffed94 Mon Sep 17 00:00:00 2001 From: julioyildo Date: Tue, 16 Mar 2021 21:53:29 +0100 Subject: [PATCH 4/8] Delete commented code --- .../ipython-bigquery/extractors.spec.ts | 93 +------------------ 1 file changed, 2 insertions(+), 91 deletions(-) diff --git a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts index 6b403cbfb..b01324950 100644 --- a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts +++ b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts @@ -1,10 +1,9 @@ import { VirtualDocument } from '../../virtual/document'; import { expect } from 'chai'; -import { foreign_code_extractors, SQL_URL_PATTERN } from './extractors'; +import { foreign_code_extractors } from './extractors'; import { extract_code, - get_the_only_virtual, - wrap_in_python_lines + get_the_only_virtual } from '../../extractors/testutils'; describe('Bigquery SQL extractors', () => { @@ -30,55 +29,6 @@ describe('Bigquery SQL extractors', () => { document.clear(); }); - /* - describe('SQL url pattern', () => { - it('matches connection strings', () => { - const correct_urls = [ - 'mysql+pymysql://scott:tiger@localhost/foo', - 'oracle://scott:tiger@127.0.0.1:1521/sidname', - 'sqlite://', - 'sqlite:///foo.db', - 'mssql+pyodbc://username:password@host/database?driver=SQL+Server+Native+Client+11.0', - 'impala://hserverhost:port/default?kerberos_service_name=hive&auth_mechanism=GSSAPI' - ]; - const pattern = new RegExp(SQL_URL_PATTERN); - for (let url of correct_urls) { - expect(pattern.test(url)).to.equal(true); - } - }); - }); -*/ -/* describe('%bigquery line magic', () => { - it('extracts simple commands', () => { - let code = wrap_in_python_lines('%bigquery select * from work'); - let { cell_code_kept, foreign_document_map } = extract(code); - - // should not be removed, but left for the static analysis (using magic overrides) - expect(cell_code_kept).to.equal(code); - let document = get_the_only_virtual(foreign_document_map); - expect(document.language).to.equal('sql'); - expect(document.value).to.equal('select * from work\n'); - });*/ -/* - it('leaves out the connection specification', () => { - let code = wrap_in_python_lines( - '%sql postgresql://will:longliveliz@localhost/shakes' - ); - let foreign_document_map = extract(code).foreign_document_map; - let document = get_the_only_virtual(foreign_document_map); - expect(document.language).to.equal('sql'); - expect(document.value).to.equal('\n'); - }); - - it('leaves out options', () => { - let code = wrap_in_python_lines('%bigquery -l'); - let foreign_document_map = extract(code).foreign_document_map; - let document = get_the_only_virtual(foreign_document_map); - expect(document.language).to.equal('sql'); - expect(document.value).to.equal('\n'); - }); - }); -*/ describe('%%bigquery cell magic', () => { it('extracts simple commands', () => { let code = "%%bigquery\nselect * from character\nwhere abbrev = 'ALICE'"; @@ -91,46 +41,7 @@ describe('Bigquery SQL extractors', () => { "select * from character\nwhere abbrev = 'ALICE'\n" ); }); -/* - it('leaves out the connection specification', () => { - let code = - "%%sql postgresql://will:longliveliz@localhost/shakes\nselect * from character\nwhere abbrev = 'ALICE'"; - let { foreign_document_map } = extract(code); - let document = get_the_only_virtual(foreign_document_map); - expect(document.language).to.equal('sql'); - expect(document.value).to.equal( - "select * from character\nwhere abbrev = 'ALICE'\n" - ); - }); - - - it('leaves out the variable assignment', () => { - let code = '%%sql works << SELECT title, year\nFROM work'; - let { foreign_document_map } = extract(code); - - let document = get_the_only_virtual(foreign_document_map); - expect(document.language).to.equal('sql'); - expect(document.value).to.equal('SELECT title, year\nFROM work\n'); - }); - - it('leaves out existing connection references', () => { - let code = '%%sql will@shakes\nSELECT title, year\nFROM work'; - let { foreign_document_map } = extract(code); - - let document = get_the_only_virtual(foreign_document_map); - expect(document.language).to.equal('sql'); - expect(document.value).to.equal('SELECT title, year\nFROM work\n'); - }); -*/ -/* it('leaves out persist option', () => { - let code = '%%bigquery --persist dataframe\nSELECT * FROM dataframe;'; - let { foreign_document_map } = extract(code); - let document = get_the_only_virtual(foreign_document_map); - expect(document.language).to.equal('sql'); - expect(document.value).to.equal('SELECT * FROM dataframe;\n'); - }); -*/ }); }); From 9049b848c006c4b4383affcc03ef210cd2badc73 Mon Sep 17 00:00:00 2001 From: julioyildo Date: Tue, 16 Mar 2021 22:43:53 +0100 Subject: [PATCH 5/8] Delete commented code --- .../src/transclusions/ipython-bigquery/extractors.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts index c7e2d564c..9932a85fc 100644 --- a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts +++ b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts @@ -31,15 +31,5 @@ export let foreign_code_extractors: IForeignCodeExtractorsRegistry = { is_standalone: true, file_extension: 'sql' }) -/* - , - new RegExpForeignCodeExtractor({ - language: 'sql', - pattern: `(?:^|\n)%bigquery (?:${SQL_URL_PATTERN}|${COMMAND_PATTERN}|(.*))\n?`, - extract_to_foreign: '$1', - is_standalone: false, - file_extension: 'sql' - }) - */ ] }; From 4bcae4bd4676a380410ad4cdaebc29b8147168b5 Mon Sep 17 00:00:00 2001 From: Serhat YILDIRIM Date: Wed, 17 Mar 2021 08:49:45 +0000 Subject: [PATCH 6/8] update transclusions, run jlpm lint --- packages/jupyterlab-lsp/src/transclusions/defaults.ts | 2 +- .../src/transclusions/ipython-bigquery/extractors.spec.ts | 7 +------ .../src/transclusions/ipython-bigquery/extractors.ts | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/jupyterlab-lsp/src/transclusions/defaults.ts b/packages/jupyterlab-lsp/src/transclusions/defaults.ts index dbbc9f298..47a051d3e 100644 --- a/packages/jupyterlab-lsp/src/transclusions/defaults.ts +++ b/packages/jupyterlab-lsp/src/transclusions/defaults.ts @@ -8,5 +8,5 @@ export const DEFAULT_TRANSCLUSIONS: JupyterFrontEndPlugin[] = [ IPYTHON_RPY2_TRANSCLUSIONS, IPYTHON_SQL_TRANSCLUSIONS, IPYTHON_TRANSCLUSIONS, - IPYTHON_BIGQUERY_TRANSCLUSIONS + IPYTHON_BIGQUERY_TRANSCLUSIONS ]; diff --git a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts index b01324950..2a4c85965 100644 --- a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts +++ b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.spec.ts @@ -1,10 +1,7 @@ import { VirtualDocument } from '../../virtual/document'; import { expect } from 'chai'; import { foreign_code_extractors } from './extractors'; -import { - extract_code, - get_the_only_virtual -} from '../../extractors/testutils'; +import { extract_code, get_the_only_virtual } from '../../extractors/testutils'; describe('Bigquery SQL extractors', () => { let document: VirtualDocument; @@ -41,7 +38,5 @@ describe('Bigquery SQL extractors', () => { "select * from character\nwhere abbrev = 'ALICE'\n" ); }); - - }); }); diff --git a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts index 9932a85fc..b9171f36e 100644 --- a/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts +++ b/packages/jupyterlab-lsp/src/transclusions/ipython-bigquery/extractors.ts @@ -10,7 +10,7 @@ const single_argument_options = [ '--use_rest_api', '--use_legacy_sql', '--verbose', - '--params' + '--params' ]; const zero_argument_options = ['-l', '--connections']; From 37d1b6da2ac3a716d4cf53f66578201f1325eb37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Sat, 20 Mar 2021 19:51:16 +0000 Subject: [PATCH 7/8] Update CHANGELOG.md --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 916f5dfdd..eb9b561d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ ## CHANGELOG -### `@krassowski/jupyterlab-lsp 3.4.2` (unreleased) +### `@krassowski/jupyterlab-lsp 3.5.0` (unreleased) + +- features: + + - adds `%%bigquery` IPython cell magic support for BigQuery ([#553], thanks @julioyildo) - bug fixes: @@ -9,6 +13,7 @@ [#544]: https://github.com/krassowski/jupyterlab-lsp/pull/544 [#547]: https://github.com/krassowski/jupyterlab-lsp/pull/547 +[#553]: https://github.com/krassowski/jupyterlab-lsp/pull/553 ### `jupyter-lsp 1.1.4` (2020-02-21) From 17d62f1b154d77b4d7ac5604fb996a03023c3714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Krassowski?= <5832902+krassowski@users.noreply.github.com> Date: Sat, 20 Mar 2021 19:52:13 +0000 Subject: [PATCH 8/8] Swap order of transclusions provider so that generic IPython is last --- packages/jupyterlab-lsp/src/transclusions/defaults.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jupyterlab-lsp/src/transclusions/defaults.ts b/packages/jupyterlab-lsp/src/transclusions/defaults.ts index 47a051d3e..97e9b93af 100644 --- a/packages/jupyterlab-lsp/src/transclusions/defaults.ts +++ b/packages/jupyterlab-lsp/src/transclusions/defaults.ts @@ -7,6 +7,6 @@ import { IPYTHON_TRANSCLUSIONS } from './ipython'; export const DEFAULT_TRANSCLUSIONS: JupyterFrontEndPlugin[] = [ IPYTHON_RPY2_TRANSCLUSIONS, IPYTHON_SQL_TRANSCLUSIONS, - IPYTHON_TRANSCLUSIONS, - IPYTHON_BIGQUERY_TRANSCLUSIONS + IPYTHON_BIGQUERY_TRANSCLUSIONS, + IPYTHON_TRANSCLUSIONS ];