From e78fc8cd038e537bcca3d494bf192270caf03f8f Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 12 Nov 2018 19:45:09 +0100 Subject: [PATCH 01/12] Test no blank lines at the end of cell for PEP8 scripts --- tests/test_read_simple_python.py | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_read_simple_python.py b/tests/test_read_simple_python.py index 52f1bf58a..dacc82223 100644 --- a/tests/test_read_simple_python.py +++ b/tests/test_read_simple_python.py @@ -499,6 +499,40 @@ def h(x): compare(script, script2) +def test_notebook_two_blank_lines_before_next_cell(script="""# + +# This is cell with a function + +def f(x): + return 4 + + +# + +# Another cell +c = 5 + + +def g(x): + return 6 + + +# + +# Final cell + +1 + 1 +"""): + notebook = jupytext.reads(script, ext='.py') + assert len(notebook.cells) == 3 + for cell in notebook.cells: + lines = cell.source.splitlines() + if len(lines) != 1: + assert lines[0] + assert lines[-1] + + script2 = jupytext.writes(notebook, ext='.py') + + compare(script, script2) + + def test_round_trip_markdown_cell_with_magic(): notebook = new_notebook(cells=[new_markdown_cell('IPython has magic commands like\n%quickref')], metadata={'jupytext': {'main_language': 'python'}}) From 121d81bdf78473158dcbe5be8446f1bbe2859374 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 12 Nov 2018 21:27:03 +0100 Subject: [PATCH 02/12] freeze_metadata option only updates jupytext metadata #124 --- jupytext/jupytext.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/jupytext/jupytext.py b/jupytext/jupytext.py index 8ff523779..34a8fe2d6 100644 --- a/jupytext/jupytext.py +++ b/jupytext/jupytext.py @@ -59,10 +59,9 @@ def reads(self, s, **_): lines = lines[pos:] if self.freeze_metadata: - metadata['jupytext'] = {'metadata_filter': { - 'notebook': {'additional': list(metadata.keys()), 'excluded': 'all'}}} - metadata['jupytext']['metadata_filter']['cells'] = { - 'additional': list(cell_metadata), 'excluded': 'all'} + metadata.setdefault('jupytext', {})['metadata_filter'] = { + 'notebook': {'additional': list(metadata.keys()), 'excluded': 'all'}, + 'cells': {'additional': list(cell_metadata), 'excluded': 'all'}} set_main_and_cell_language(metadata, cells, self.format.extension) From c103eb1d20bd48f774dfc5c4b7f6cad7b52f900c Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 12 Nov 2018 21:27:43 +0100 Subject: [PATCH 03/12] Do not create an empty 'jupytext' metadata section --- jupytext/contentsmanager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jupytext/contentsmanager.py b/jupytext/contentsmanager.py index 581637765..2a4899237 100644 --- a/jupytext/contentsmanager.py +++ b/jupytext/contentsmanager.py @@ -341,7 +341,7 @@ def get(self, path, content=True, type=None, format=None, except OverflowError: pass - jupytext_metadata = model['content']['metadata'].setdefault('jupytext', {}) + jupytext_metadata = model['content']['metadata'].get('jupytext', {}) if self.default_notebook_metadata_filter: (jupytext_metadata.setdefault('metadata_filter', {}) .setdefault('notebook', self.default_notebook_metadata_filter)) @@ -354,6 +354,9 @@ def get(self, path, content=True, type=None, format=None, if filter is not None: jupytext_metadata['metadata_filter'][filter_level] = metadata_filter_as_dict(filter) + if jupytext_metadata: + model['content']['metadata']['jupytext'] = jupytext_metadata + if model_outputs: combine_inputs_with_outputs(model['content'], model_outputs['content']) elif not fmt.endswith('.ipynb'): From 433406aa6e00c27655ef2d253a2663ca06eaa76a Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 12 Nov 2018 21:41:05 +0100 Subject: [PATCH 04/12] Jupytext cell metadata from source, not ipynb --- jupytext/cell_metadata.py | 12 +++++++----- jupytext/combine.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/jupytext/cell_metadata.py b/jupytext/cell_metadata.py index adfe9342e..835c07e11 100644 --- a/jupytext/cell_metadata.py +++ b/jupytext/cell_metadata.py @@ -26,14 +26,16 @@ _BOOLEAN_OPTIONS_DICTIONARY = [('hide_input', 'echo', True), ('hide_output', 'include', True)] -_IGNORE_CELL_METADATA = ','.join('-{}'.format(name) for name in [ - # Frequent cell metadata that should not enter the text representation - # (these metadata are preserved in the paired Jupyter notebook). - 'autoscroll', 'collapsed', 'scrolled', 'trusted', 'ExecuteTime', +_JUPYTEXT_CELL_METADATA = [ # Pre-jupytext metadata 'skipline', 'noskipline', # Jupytext metadata - 'cell_marker', 'lines_to_next_cell', 'lines_to_end_of_cell_marker']) + 'cell_marker', 'lines_to_next_cell', 'lines_to_end_of_cell_marker'] +_IGNORE_CELL_METADATA = ','.join('-{}'.format(name) for name in [ + # Frequent cell metadata that should not enter the text representation + # (these metadata are preserved in the paired Jupyter notebook). + 'autoscroll', 'collapsed', 'scrolled', 'trusted', 'ExecuteTime'] + + _JUPYTEXT_CELL_METADATA) _PERCENT_CELL = re.compile( r'(# |#)%%([^\{\[]*)(|\[raw\]|\[markdown\])([^\{\[]*)(|\{.*\})\s*$') diff --git a/jupytext/combine.py b/jupytext/combine.py index 84cb5654b..f49291abc 100644 --- a/jupytext/combine.py +++ b/jupytext/combine.py @@ -2,7 +2,7 @@ """ import re from copy import copy -from .cell_metadata import _IGNORE_CELL_METADATA +from .cell_metadata import _IGNORE_CELL_METADATA, _JUPYTEXT_CELL_METADATA from .header import _DEFAULT_NOTEBOOK_METADATA from .metadata_filter import filter_metadata @@ -58,7 +58,7 @@ def combine_inputs_with_outputs(nb_source, nb_outputs): _IGNORE_CELL_METADATA) for key in ocell.metadata: - if key not in ocell_filtered_metadata: + if key not in ocell_filtered_metadata and key not in _JUPYTEXT_CELL_METADATA: cell.metadata[key] = ocell.metadata[key] output_code_cells = output_code_cells[(i + 1):] From c6b8fd1762361c33581c1398447b4cbd10af5b61 Mon Sep 17 00:00:00 2001 From: kiendang Date: Sun, 11 Nov 2018 08:53:37 +0800 Subject: [PATCH 05/12] Add .r extension for R files --- jupytext/languages.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jupytext/languages.py b/jupytext/languages.py index 8da4b4700..f57340858 100644 --- a/jupytext/languages.py +++ b/jupytext/languages.py @@ -7,6 +7,7 @@ _SCRIPT_EXTENSIONS = {'.py': {'language': 'python', 'comment': '#'}, '.R': {'language': 'R', 'comment': '#'}, + '.r': {'language': 'R', 'comment': '#'}, '.jl': {'language': 'julia', 'comment': '#'}, '.cpp': {'language': 'c++', 'comment': '//'}, '.ss': {'language': 'scheme', 'comment': ';;'}} From fe5bc585603343b37c514d22420feb4a5548439a Mon Sep 17 00:00:00 2001 From: kiendang Date: Sun, 11 Nov 2018 23:27:50 +0800 Subject: [PATCH 06/12] Enable paired notebooks and conversion for .r files --- jupytext/formats.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jupytext/formats.py b/jupytext/formats.py index 9da337da3..c4aac0aba 100644 --- a/jupytext/formats.py +++ b/jupytext/formats.py @@ -63,6 +63,14 @@ def __init__(self, cell_reader_class=RScriptCellReader, cell_exporter_class=RScriptCellExporter, # Version 1.0 on 2018-08-22 - jupytext v0.5.2 : Initial version + current_version_number='1.0'), + + NotebookFormatDescription( + format_name='spin', + extension='.r', + header_prefix="#'", + cell_reader_class=RScriptCellReader, + cell_exporter_class=RScriptCellExporter, current_version_number='1.0')] + \ [ NotebookFormatDescription( From 074339e7fd3ece72cb3552d0291aac01fbf7a2c0 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 12 Nov 2018 22:36:42 +0100 Subject: [PATCH 07/12] Support for .r files --- jupytext/cell_reader.py | 2 +- jupytext/formats.py | 23 +++++------- jupytext/version.py | 2 +- .../mirror/ipynb_to_percent/ir_notebook.low.r | 26 ++++++++++++++ .../mirror/ipynb_to_script/ir_notebook.low.r | 22 ++++++++++++ .../mirror/ipynb_to_spin/ir_notebook.R | 22 ++++++++++++ .../mirror/ipynb_to_spin/ir_notebook.low.r | 22 ++++++++++++ tests/test_escape_magics.py | 6 ++-- tests/test_ipynb_to_R.py | 9 ++--- tests/test_mirror.py | 20 +++++++++++ tests/test_read_simple_R.py | 36 +++++++++++-------- 11 files changed, 152 insertions(+), 38 deletions(-) create mode 100644 tests/notebooks/mirror/ipynb_to_percent/ir_notebook.low.r create mode 100644 tests/notebooks/mirror/ipynb_to_script/ir_notebook.low.r create mode 100644 tests/notebooks/mirror/ipynb_to_spin/ir_notebook.R create mode 100644 tests/notebooks/mirror/ipynb_to_spin/ir_notebook.low.r diff --git a/jupytext/cell_reader.py b/jupytext/cell_reader.py index b2da8c15f..b8d92dce2 100644 --- a/jupytext/cell_reader.py +++ b/jupytext/cell_reader.py @@ -203,7 +203,7 @@ def find_cell_content(self, lines): if not is_active(self.ext, self.metadata) or \ ('active' not in self.metadata and self.language and self.language != self.default_language): - self.content = uncomment(source, self.comment if self.ext != '.R' else '#') + self.content = uncomment(source, self.comment if self.ext not in ['.r', '.R'] else '#') else: self.content = self.uncomment_code_and_magics(source) diff --git a/jupytext/formats.py b/jupytext/formats.py index c4aac0aba..c3f6d65fb 100644 --- a/jupytext/formats.py +++ b/jupytext/formats.py @@ -54,24 +54,16 @@ def __init__(self, cell_reader_class=RMarkdownCellReader, cell_exporter_class=RMarkdownCellExporter, # Version 1.0 on 2018-08-22 - jupytext v0.5.2 : Initial version - current_version_number='1.0'), - + current_version_number='1.0')] + \ + [ NotebookFormatDescription( format_name='spin', - extension='.R', + extension=ext, header_prefix="#'", cell_reader_class=RScriptCellReader, cell_exporter_class=RScriptCellExporter, # Version 1.0 on 2018-08-22 - jupytext v0.5.2 : Initial version - current_version_number='1.0'), - - NotebookFormatDescription( - format_name='spin', - extension='.r', - header_prefix="#'", - cell_reader_class=RScriptCellReader, - cell_exporter_class=RScriptCellExporter, - current_version_number='1.0')] + \ + current_version_number='1.0') for ext in ['.r', '.R']] + \ [ NotebookFormatDescription( format_name='light', @@ -156,7 +148,7 @@ def read_metadata(text, ext): comment = _SCRIPT_EXTENSIONS.get(ext, {}).get('comment', '#') metadata, _, _ = header_to_metadata_and_cell(lines, comment) - if ext == '.R' and not metadata: + if ext in ['.r', '.R'] and not metadata: metadata, _, _ = header_to_metadata_and_cell(lines, "#'") return metadata @@ -190,7 +182,7 @@ def guess_format(text, ext): twenty_hash_count = 0 double_percent_count = 0 - parser = StringParser(language='R' if ext == '.R' else 'python') + parser = StringParser(language='R' if ext in ['.r', '.R'] else 'python') for line in lines: parser.read_line(line) if parser.is_quoted(): @@ -198,7 +190,8 @@ def guess_format(text, ext): # Don't count escaped Jupyter magics (no space between # %% and command) as cells - if double_percent_re.match(line) or double_percent_and_space_re.match(line) or nbconvert_script_re.match(line): + if double_percent_re.match(line) or double_percent_and_space_re.match(line) or \ + nbconvert_script_re.match(line): double_percent_count += 1 if line.startswith(twenty_hash) and ext == '.py': diff --git a/jupytext/version.py b/jupytext/version.py index 92fc5b6eb..4dcf642b0 100644 --- a/jupytext/version.py +++ b/jupytext/version.py @@ -1,3 +1,3 @@ """Jupytext's version number""" -__version__ = '0.8.4' +__version__ = '0.8.5' diff --git a/tests/notebooks/mirror/ipynb_to_percent/ir_notebook.low.r b/tests/notebooks/mirror/ipynb_to_percent/ir_notebook.low.r new file mode 100644 index 000000000..e8ec06ca9 --- /dev/null +++ b/tests/notebooks/mirror/ipynb_to_percent/ir_notebook.low.r @@ -0,0 +1,26 @@ +# --- +# jupyter: +# kernelspec: +# display_name: R +# language: R +# name: ir +# language_info: +# codemirror_mode: r +# file_extension: .r +# mimetype: text/x-r-source +# name: R +# pygments_lexer: r +# version: 3.5.0 +# --- + +# %% [markdown] +# This is a jupyter notebook that uses the IR kernel. + +# %% +sum(1:10) + +# %% +plot(cars) + +# %% + diff --git a/tests/notebooks/mirror/ipynb_to_script/ir_notebook.low.r b/tests/notebooks/mirror/ipynb_to_script/ir_notebook.low.r new file mode 100644 index 000000000..642d0bf7c --- /dev/null +++ b/tests/notebooks/mirror/ipynb_to_script/ir_notebook.low.r @@ -0,0 +1,22 @@ +#' --- +#' jupyter: +#' kernelspec: +#' display_name: R +#' language: R +#' name: ir +#' language_info: +#' codemirror_mode: r +#' file_extension: .r +#' mimetype: text/x-r-source +#' name: R +#' pygments_lexer: r +#' version: 3.5.0 +#' --- + +#' This is a jupyter notebook that uses the IR kernel. + +sum(1:10) + +plot(cars) + + diff --git a/tests/notebooks/mirror/ipynb_to_spin/ir_notebook.R b/tests/notebooks/mirror/ipynb_to_spin/ir_notebook.R new file mode 100644 index 000000000..642d0bf7c --- /dev/null +++ b/tests/notebooks/mirror/ipynb_to_spin/ir_notebook.R @@ -0,0 +1,22 @@ +#' --- +#' jupyter: +#' kernelspec: +#' display_name: R +#' language: R +#' name: ir +#' language_info: +#' codemirror_mode: r +#' file_extension: .r +#' mimetype: text/x-r-source +#' name: R +#' pygments_lexer: r +#' version: 3.5.0 +#' --- + +#' This is a jupyter notebook that uses the IR kernel. + +sum(1:10) + +plot(cars) + + diff --git a/tests/notebooks/mirror/ipynb_to_spin/ir_notebook.low.r b/tests/notebooks/mirror/ipynb_to_spin/ir_notebook.low.r new file mode 100644 index 000000000..642d0bf7c --- /dev/null +++ b/tests/notebooks/mirror/ipynb_to_spin/ir_notebook.low.r @@ -0,0 +1,22 @@ +#' --- +#' jupyter: +#' kernelspec: +#' display_name: R +#' language: R +#' name: ir +#' language_info: +#' codemirror_mode: r +#' file_extension: .r +#' mimetype: text/x-r-source +#' name: R +#' pygments_lexer: r +#' version: 3.5.0 +#' --- + +#' This is a jupyter notebook that uses the IR kernel. + +sum(1:10) + +plot(cars) + + diff --git a/tests/test_escape_magics.py b/tests/test_escape_magics.py index 30e2a9524..f46ff4818 100644 --- a/tests/test_escape_magics.py +++ b/tests/test_escape_magics.py @@ -57,7 +57,8 @@ def test_magics_are_commented(ext_and_format_name): ext, format_name = parse_one_format(ext_and_format_name) nb = new_notebook(cells=[new_code_cell('%pylab inline')], metadata={'jupytext': {'comment_magics': True, - 'main_language': 'R' if ext == '.R' else 'scheme' if ext == '.ss' else 'python'}}) + 'main_language': 'R' if ext in ['.r', '.R'] + else 'scheme' if ext == '.ss' else 'python'}}) text = jupytext.writes(nb, ext, format_name) assert '%pylab inline' not in text.splitlines() @@ -75,7 +76,8 @@ def test_magics_are_not_commented(ext_and_format_name): ext, format_name = parse_one_format(ext_and_format_name) nb = new_notebook(cells=[new_code_cell('%pylab inline')], metadata={'jupytext': {'comment_magics': False, - 'main_language': 'R' if ext == '.R' else 'scheme' if ext == '.ss' else 'python'}}) + 'main_language': 'R' if ext in ['.r', '.R'] + else 'scheme' if ext == '.ss' else 'python'}}) text = jupytext.writes(nb, ext, format_name) assert '%pylab inline' in text.splitlines() diff --git a/tests/test_ipynb_to_R.py b/tests/test_ipynb_to_R.py index b1a1146fa..12e624131 100644 --- a/tests/test_ipynb_to_R.py +++ b/tests/test_ipynb_to_R.py @@ -1,4 +1,5 @@ import nbformat +import itertools import pytest import jupytext from jupytext.compare import compare_notebooks @@ -7,8 +8,8 @@ jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER = False -@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_R')) -def test_identity_source_write_read(nb_file): +@pytest.mark.parametrize('nb_file,ext', itertools.product(list_notebooks('ipynb_R'), ['.r', '.R'])) +def test_identity_source_write_read(nb_file, ext): """ Test that writing the notebook with R, and read again, is the same as removing outputs @@ -17,7 +18,7 @@ def test_identity_source_write_read(nb_file): with open(nb_file) as fp: nb1 = nbformat.read(fp, as_version=4) - R = jupytext.writes(nb1, ext='.R') - nb2 = jupytext.reads(R, ext='.R') + R = jupytext.writes(nb1, ext=ext) + nb2 = jupytext.reads(R, ext=ext) compare_notebooks(nb1, nb2) diff --git a/tests/test_mirror.py b/tests/test_mirror.py index 6693406ee..e08cba11b 100644 --- a/tests/test_mirror.py +++ b/tests/test_mirror.py @@ -98,6 +98,11 @@ def test_ipynb_to_R(nb_file): assert_conversion_same_as_mirror(nb_file, '.R', 'ipynb_to_script') +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_R')) +def test_ipynb_to_r(nb_file): + assert_conversion_same_as_mirror(nb_file, '.low.r', 'ipynb_to_script') + + @pytest.mark.parametrize('nb_file', list_notebooks('ipynb_scheme')) def test_ipynb_to_scheme(nb_file): assert_conversion_same_as_mirror(nb_file, '.ss', 'ipynb_to_script') @@ -123,6 +128,21 @@ def test_ipynb_to_R_percent(nb_file): assert_conversion_same_as_mirror(nb_file, '.R', 'ipynb_to_percent', format_name='percent') +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_R')) +def test_ipynb_to_r_percent(nb_file): + assert_conversion_same_as_mirror(nb_file, '.low.r', 'ipynb_to_percent', format_name='percent') + + +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_R')) +def test_ipynb_to_R_spin(nb_file): + assert_conversion_same_as_mirror(nb_file, '.R', 'ipynb_to_spin', format_name='spin') + + +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_R')) +def test_ipynb_to_r_spin(nb_file): + assert_conversion_same_as_mirror(nb_file, '.low.r', 'ipynb_to_spin', format_name='spin') + + @pytest.mark.parametrize('nb_file', list_notebooks('ipynb_cpp')) def test_ipynb_to_cpp_percent(nb_file): assert_conversion_same_as_mirror(nb_file, '.cpp', 'ipynb_to_percent', format_name='percent') diff --git a/tests/test_read_simple_R.py b/tests/test_read_simple_R.py index 8c49994be..c0561457e 100644 --- a/tests/test_read_simple_R.py +++ b/tests/test_read_simple_R.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- +import pytest from testfixtures import compare import jupytext jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER = False -def test_read_simple_file(rnb="""#' --- +@pytest.mark.parametrize('ext', ['.r', '.R']) +def test_read_simple_file(ext, rnb="""#' --- #' title: Simple file #' --- @@ -21,7 +23,7 @@ def test_read_simple_file(rnb="""#' --- h <- function(y) y + 1 """): - nb = jupytext.reads(rnb, ext='.R') + nb = jupytext.reads(rnb, ext=ext) assert len(nb.cells) == 4 assert nb.cells[0].cell_type == 'raw' assert nb.cells[0].source == '---\ntitle: Simple file\n---' @@ -36,11 +38,12 @@ def test_read_simple_file(rnb="""#' --- compare(nb.cells[3].source, '''h <- function(y) y + 1''') - rnb2 = jupytext.writes(nb, ext='.R') + rnb2 = jupytext.writes(nb, ext=ext) compare(rnb, rnb2) -def test_read_less_simple_file(rnb="""#' --- +@pytest.mark.parametrize('ext', ['.r', '.R']) +def test_read_less_simple_file(ext, rnb="""#' --- #' title: Less simple file #' --- @@ -58,7 +61,7 @@ def test_read_less_simple_file(rnb="""#' --- return(y-1) } """): - nb = jupytext.reads(rnb, ext='.R') + nb = jupytext.reads(rnb, ext=ext) assert len(nb.cells) == 4 assert nb.cells[0].cell_type == 'raw' @@ -79,11 +82,12 @@ def test_read_less_simple_file(rnb="""#' --- return(y-1) }''') - rnb2 = jupytext.writes(nb, ext='.R') + rnb2 = jupytext.writes(nb, ext=ext) compare(rnb, rnb2) -def test_no_space_after_code(rnb=u"""# -*- coding: utf-8 -*- +@pytest.mark.parametrize('ext', ['.r', '.R']) +def test_no_space_after_code(ext, rnb=u"""# -*- coding: utf-8 -*- #' Markdown cell f <- function(x) @@ -93,7 +97,7 @@ def test_no_space_after_code(rnb=u"""# -*- coding: utf-8 -*- #' And a new cell, and non ascii contênt """): - nb = jupytext.reads(rnb, ext='.R') + nb = jupytext.reads(rnb, ext=ext) assert len(nb.cells) == 3 assert nb.cells[0].cell_type == 'markdown' @@ -106,20 +110,22 @@ def test_no_space_after_code(rnb=u"""# -*- coding: utf-8 -*- assert nb.cells[2].cell_type == 'markdown' assert nb.cells[2].source == u'And a new cell, and non ascii contênt' - rnb2 = jupytext.writes(nb, ext='.R') + rnb2 = jupytext.writes(nb, ext=ext) compare(rnb, rnb2) -def test_read_write_script(rnb="""#!/usr/bin/env Rscript +@pytest.mark.parametrize('ext', ['.r', '.R']) +def test_read_write_script(ext, rnb="""#!/usr/bin/env Rscript # coding=utf-8 print('Hello world') """): - nb = jupytext.reads(rnb, ext='.R') - rnb2 = jupytext.writes(nb, ext='.R') + nb = jupytext.reads(rnb, ext=ext) + rnb2 = jupytext.writes(nb, ext=ext) compare(rnb, rnb2) -def test_escape_start_pattern(rnb="""#' The code start pattern '#+' can +@pytest.mark.parametrize('ext', ['.r', '.R']) +def test_escape_start_pattern(ext, rnb="""#' The code start pattern '#+' can #' appear in code and markdown cells. #' In markdown cells it is escaped like here: @@ -129,7 +135,7 @@ def test_escape_start_pattern(rnb="""#' The code start pattern '#+' can # #+ cell_name language="python" 1 + 1 """): - nb = jupytext.reads(rnb, ext='.R') + nb = jupytext.reads(rnb, ext=ext) assert len(nb.cells) == 3 assert nb.cells[0].cell_type == 'markdown' assert nb.cells[1].cell_type == 'markdown' @@ -140,5 +146,5 @@ def test_escape_start_pattern(rnb="""#' The code start pattern '#+' can '''# In code cells like this one, it is also escaped #+ cell_name language="python" 1 + 1''') - rnb2 = jupytext.writes(nb, ext='.R') + rnb2 = jupytext.writes(nb, ext=ext) compare(rnb, rnb2) From 4c3a8bfaebb23a9fd6135c349a59c7a441de89da Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 12 Nov 2018 22:43:47 +0100 Subject: [PATCH 08/12] Bash scripts as notebooks #127 --- jupytext/languages.py | 3 +- .../ipynb_bash/sample_bash_notebook.ipynb | 70 +++++++++++++++++++ .../ipynb_to_percent/sample_bash_notebook.sh | 22 ++++++ .../ipynb_to_script/sample_bash_notebook.sh | 19 +++++ tests/test_mirror.py | 10 +++ 5 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 tests/notebooks/ipynb_bash/sample_bash_notebook.ipynb create mode 100644 tests/notebooks/mirror/ipynb_to_percent/sample_bash_notebook.sh create mode 100644 tests/notebooks/mirror/ipynb_to_script/sample_bash_notebook.sh diff --git a/jupytext/languages.py b/jupytext/languages.py index f57340858..8779520c6 100644 --- a/jupytext/languages.py +++ b/jupytext/languages.py @@ -10,7 +10,8 @@ '.r': {'language': 'R', 'comment': '#'}, '.jl': {'language': 'julia', 'comment': '#'}, '.cpp': {'language': 'c++', 'comment': '//'}, - '.ss': {'language': 'scheme', 'comment': ';;'}} + '.ss': {'language': 'scheme', 'comment': ';;'}, + '.sh': {'language': 'bash', 'comment': '#'}} def default_language_from_metadata_and_ext(notebook, ext): diff --git a/tests/notebooks/ipynb_bash/sample_bash_notebook.ipynb b/tests/notebooks/ipynb_bash/sample_bash_notebook.ipynb new file mode 100644 index 000000000..c9d0bd24c --- /dev/null +++ b/tests/notebooks/ipynb_bash/sample_bash_notebook.ipynb @@ -0,0 +1,70 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " \u001b[0m\u001b[01;32mREADME.md\u001b[0m \u001b[01;32m'Sample bash notebook.ipynb'\u001b[0m \u001b[34;42mbinder\u001b[0m \u001b[34;42mdemo\u001b[0m \u001b[34;42mnotebook\u001b[0m \u001b[34;42mslides\u001b[0m\n" + ] + } + ], + "source": [ + "ls" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# https://coderwall.com/p/euwpig/a-better-git-log\n", + "git config --global alias.lg \"log --color --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* \u001b[31m64accf1\u001b[m -\u001b[33m (HEAD -> master, origin/master, origin/HEAD)\u001b[m Slides reviewed \u001b[32m(23 hours ago) \u001b[1;34m\u001b[m\n", + "* \u001b[31mbf3d0ab\u001b[m -\u001b[33m\u001b[m README \u001b[32m(23 hours ago) \u001b[1;34m\u001b[m\n", + "* \u001b[31md1116f6\u001b[m -\u001b[33m\u001b[m Use Binder \u001b[32m(23 hours ago) \u001b[1;34m\u001b[m\n", + "* \u001b[31m3526fc6\u001b[m -\u001b[33m\u001b[m Demo notebook has data in the metric explorer \u001b[32m(23 hours ago) \u001b[1;34m\u001b[m\n", + "* \u001b[31mf4139fd\u001b[m -\u001b[33m\u001b[m Ignore more patterns \u001b[32m(23 hours ago) \u001b[1;34m\u001b[m\n", + "* \u001b[31m984ccd1\u001b[m -\u001b[33m\u001b[m Demo script updated \u001b[32m(2 days ago) \u001b[1;34m\u001b[m\n", + "* \u001b[31m6874a22\u001b[m -\u001b[33m\u001b[m Updated final notebook \u001b[32m(2 days ago) \u001b[1;34m\u001b[m\n", + "* \u001b[31ma6b8f88\u001b[m -\u001b[33m\u001b[m Slides and demo are getting ready \u001b[32m(4 days ago) \u001b[1;34m\u001b[m" + ] + } + ], + "source": [ + "git lg" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Bash", + "language": "bash", + "name": "bash" + }, + "language_info": { + "codemirror_mode": "shell", + "file_extension": ".sh", + "mimetype": "text/x-sh", + "name": "bash" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tests/notebooks/mirror/ipynb_to_percent/sample_bash_notebook.sh b/tests/notebooks/mirror/ipynb_to_percent/sample_bash_notebook.sh new file mode 100644 index 000000000..ddd3275fb --- /dev/null +++ b/tests/notebooks/mirror/ipynb_to_percent/sample_bash_notebook.sh @@ -0,0 +1,22 @@ +# --- +# jupyter: +# kernelspec: +# display_name: Bash +# language: bash +# name: bash +# language_info: +# codemirror_mode: shell +# file_extension: .sh +# mimetype: text/x-sh +# name: bash +# --- + +# %% +ls + +# %% +# https://coderwall.com/p/euwpig/a-better-git-log +git config --global alias.lg "log --color --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit" + +# %% +git lg diff --git a/tests/notebooks/mirror/ipynb_to_script/sample_bash_notebook.sh b/tests/notebooks/mirror/ipynb_to_script/sample_bash_notebook.sh new file mode 100644 index 000000000..34250cd85 --- /dev/null +++ b/tests/notebooks/mirror/ipynb_to_script/sample_bash_notebook.sh @@ -0,0 +1,19 @@ +# --- +# jupyter: +# kernelspec: +# display_name: Bash +# language: bash +# name: bash +# language_info: +# codemirror_mode: shell +# file_extension: .sh +# mimetype: text/x-sh +# name: bash +# --- + +ls + +# https://coderwall.com/p/euwpig/a-better-git-log +git config --global alias.lg "log --color --graph --pretty=format:'%Cred%h%Creset -%C(yellow)%d%Creset %s %Cgreen(%cr) %C(bold blue)<%an>%Creset' --abbrev-commit" + +git lg diff --git a/tests/test_mirror.py b/tests/test_mirror.py index e08cba11b..49f4e4247 100644 --- a/tests/test_mirror.py +++ b/tests/test_mirror.py @@ -108,6 +108,11 @@ def test_ipynb_to_scheme(nb_file): assert_conversion_same_as_mirror(nb_file, '.ss', 'ipynb_to_script') +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_bash')) +def test_ipynb_to_bash(nb_file): + assert_conversion_same_as_mirror(nb_file, '.sh', 'ipynb_to_script') + + @pytest.mark.parametrize('nb_file', list_notebooks('ipynb_cpp')) def test_ipynb_to_cpp(nb_file): assert_conversion_same_as_mirror(nb_file, '.cpp', 'ipynb_to_script') @@ -153,6 +158,11 @@ def test_ipynb_to_scheme_percent(nb_file): assert_conversion_same_as_mirror(nb_file, '.ss', 'ipynb_to_percent', format_name='percent') +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_bash')) +def test_ipynb_to_bash_percent(nb_file): + assert_conversion_same_as_mirror(nb_file, '.sh', 'ipynb_to_percent', format_name='percent') + + @pytest.mark.parametrize('nb_file', list_notebooks('percent')) def test_percent_to_ipynb(nb_file): assert_conversion_same_as_mirror(nb_file, '.ipynb', 'script_to_ipynb', format_name='percent') From 5f9a532d76f4c31f5e13a66e8e38902f5368a6e5 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 12 Nov 2018 23:30:01 +0100 Subject: [PATCH 09/12] Default to first kernel that matches the language #120 --- jupytext/contentsmanager.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/jupytext/contentsmanager.py b/jupytext/contentsmanager.py index 2a4899237..57eaa17e0 100644 --- a/jupytext/contentsmanager.py +++ b/jupytext/contentsmanager.py @@ -15,6 +15,7 @@ pass from notebook.services.contents.filemanager import FileContentsManager +from jupyter_client.kernelspec import find_kernel_specs, get_kernel_spec import jupytext from .combine import combine_inputs_with_outputs @@ -23,6 +24,15 @@ from .metadata_filter import metadata_filter_as_dict +def kernelspec_from_language(language): + """Return the kernel specification for the first kernel with a matching language""" + for name in find_kernel_specs(): + ks = get_kernel_spec(name) + if ks.language == language: + return {'name': name, 'language': language, 'display_name': ks.display_name} + return None + + def _jupytext_writes(ext, format_name): def _writes(nbk, version=nbformat.NO_CONVERT, **kwargs): return jupytext.writes(nbk, version=version, ext=ext, format_name=format_name, **kwargs) @@ -360,8 +370,15 @@ def get(self, path, content=True, type=None, format=None, if model_outputs: combine_inputs_with_outputs(model['content'], model_outputs['content']) elif not fmt.endswith('.ipynb'): - self.notary.sign(model['content']) - self.mark_trusted_cells(model['content'], path) + nb = model['content'] + language = nb.metadata.get('jupytext', {}).get('main_language', 'python') + if 'kernelspec' not in nb.metadata and language != 'python': + kernelspec = kernelspec_from_language(language) + if kernelspec: + nb.metadata['kernelspec'] = kernelspec + + self.notary.sign(nb) + self.mark_trusted_cells(nb, path) return model From cb8fd8c845549738a31fd9a26a7eb8764f0c2caa Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 12 Nov 2018 23:34:28 +0100 Subject: [PATCH 10/12] Version 0.8.5 --- HISTORY.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/HISTORY.rst b/HISTORY.rst index cfa724c60..db95f0b22 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,15 @@ Release History --------------- +0.8.5 (2018-11-13) +++++++++++++++++++++++ + +**Improvements** + +- ``bash`` scripts as notebooks (#127) +- R scripts with ``.r`` extension are supported (#122) +- Jupytext selects the first kernel that matches the language (#120) + 0.8.4 (2018-10-29) ++++++++++++++++++++++ From 598f2c992edf118926802bfa199c786cf7199ae4 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Mon, 12 Nov 2018 23:51:44 +0100 Subject: [PATCH 11/12] Failsafe on kernelspec errors #120 KeyError seen on CI: https://travis-ci.com/mwouts/jupytext/jobs/158053471 --- jupytext/contentsmanager.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/jupytext/contentsmanager.py b/jupytext/contentsmanager.py index 57eaa17e0..1c8ddb397 100644 --- a/jupytext/contentsmanager.py +++ b/jupytext/contentsmanager.py @@ -26,10 +26,13 @@ def kernelspec_from_language(language): """Return the kernel specification for the first kernel with a matching language""" - for name in find_kernel_specs(): - ks = get_kernel_spec(name) - if ks.language == language: - return {'name': name, 'language': language, 'display_name': ks.display_name} + try: + for name in find_kernel_specs(): + ks = get_kernel_spec(name) + if ks.language == language: + return {'name': name, 'language': language, 'display_name': ks.display_name} + except KeyError: + pass return None From 80d230d5c2d3b5a31f6307926b929d972ad8b8be Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Tue, 13 Nov 2018 00:05:05 +0100 Subject: [PATCH 12/12] Failsafe on kernelspec errors #120 https://travis-ci.com/mwouts/jupytext/jobs/158056709 --- jupytext/contentsmanager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupytext/contentsmanager.py b/jupytext/contentsmanager.py index 1c8ddb397..b5302146b 100644 --- a/jupytext/contentsmanager.py +++ b/jupytext/contentsmanager.py @@ -31,7 +31,7 @@ def kernelspec_from_language(language): ks = get_kernel_spec(name) if ks.language == language: return {'name': name, 'language': language, 'display_name': ks.display_name} - except KeyError: + except (KeyError, ValueError): pass return None