diff --git a/.travis.yml b/.travis.yml index 11b9dd17a..32747e9b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ install: - pip install . before_script: # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude=ipynb_to_percent + - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics script: diff --git a/HISTORY.rst b/HISTORY.rst index 64a76936b..1ed1f6bb2 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -10,6 +10,8 @@ Release History - The ``language_info`` section is not part of the default header any more. Language information is now taken from metadata ``kernelspec.language``. (#105). - When opening a paired notebook, the active file is now the file that was originally opened (#118). When saving a notebook, timestamps of all the alternative representations are tested to ensure that Jupyter's autosave does not override manual modifications. +- Jupyter magic commands are now commented per default in the ``percent`` format (#126, #132). Version for the ``percent`` format increases from '1.1' to '1.2'. Set an option ``comment_magics`` to ``false`` either per notebook, or globally on Jupytext's contents manager, or on `jupytext`'s command line, if you prefer not to comment Jupyter magics. + 0.8.5 (2018-11-13) ++++++++++++++++++++++ diff --git a/README.md b/README.md index 325f0a483..c6a23ca2f 100755 --- a/README.md +++ b/README.md @@ -138,6 +138,7 @@ The package provides a `jupytext` script for command line conversion between the ```bash jupytext --to python notebook.ipynb # create a notebook.py file jupytext --to py:percent notebook.ipynb # create a notebook.py file in the double percent format +jupytext --to py:percent --comment-magics false notebook.ipynb # create a notebook.py file in the double percent format, and do not comment magic commands jupytext --to markdown notebook.ipynb # create a notebook.md file jupytext --output script.py notebook.ipynb # create a script.py file @@ -282,7 +283,7 @@ That being said, Jupytext also works well from Jupyter Lab. Please note that: ## Will my notebook really run in an IDE? -Well, that's what we expect. There's however a big difference in the python environments between Python IDEs and Jupyter: in most IDEs the code is executed with `python` and not in a Jupyter kernel. For this reason, `jupytext` comments Jupyter magics found in your notebook when exporting to R Markdown, and to scripts in all format but the `percent` one. Magics are not commented in the plain Markdown representation, nor in the `percent` format, as some editors use that format in combination with Jupyter kernels. Change this by adding a `#escape` or `#noescape` flag on the same line as the magic, or a `"comment_magics": true` or `false` entry in the notebook metadata, in the `"jupytext"` section. Or set your preference globally on the contents manager by adding this line to `.jupyter/jupyter_notebook_config.py`: +Well, that's what we expect. There's however a big difference in the python environments between Python IDEs and Jupyter: in most IDEs the code is executed with `python` and not in a Jupyter kernel. For this reason, `jupytext` comments Jupyter magics found in your notebook when exporting to all format but the plain Markdown one. Change this by adding a `#escape` or `#noescape` flag on the same line as the magic, or a `"comment_magics": true` or `false` entry in the notebook metadata, in the `"jupytext"` section. Or set your preference globally on the contents manager by adding this line to `.jupyter/jupyter_notebook_config.py`: ```python c.ContentsManager.comment_magics = True # or False ``` diff --git a/jupytext/cell_reader.py b/jupytext/cell_reader.py index b8d92dce2..8b8b12ea8 100644 --- a/jupytext/cell_reader.py +++ b/jupytext/cell_reader.py @@ -408,7 +408,7 @@ def find_cell_end(self, lines): class DoublePercentScriptCellReader(ScriptCellReader): """Read notebook cells from Hydrogen/Spyder/VScode scripts (#59)""" - default_comment_magics = False + default_comment_magics = True def __init__(self, ext, comment_magics=None): ScriptCellReader.__init__(self, ext, comment_magics) @@ -443,9 +443,10 @@ def find_cell_content(self, lines): # Cell content source = lines[cell_start:cell_end_marker] - if self.cell_type != 'code' or (self.metadata and not is_active('py', self.metadata)): + if self.cell_type != 'code' or (self.metadata and not is_active('py', self.metadata)) \ + or (self.language is not None and self.language != self.default_language): source = uncomment(source, self.comment) - if self.comment_magics: + elif self.metadata is not None and self.comment_magics: source = self.uncomment_code_and_magics(source) self.content = source diff --git a/jupytext/cell_to_text.py b/jupytext/cell_to_text.py index 045aa163f..770947647 100644 --- a/jupytext/cell_to_text.py +++ b/jupytext/cell_to_text.py @@ -60,8 +60,7 @@ def __init__(self, cell, default_language, ext, comment_magics=None, cell_metada # how many blank lines before next cell self.lines_to_next_cell = cell.metadata.get('lines_to_next_cell', 1) - self.lines_to_end_of_cell_marker = \ - cell.metadata.get('lines_to_end_of_cell_marker', 0) + self.lines_to_end_of_cell_marker = cell.metadata.get('lines_to_end_of_cell_marker', 0) # for compatibility with v0.5.4 and lower (to be removed) if 'skipline' in cell.metadata: @@ -286,8 +285,8 @@ def code_to_text(self): class DoublePercentCellExporter(BaseCellExporter): """A class that can represent a notebook cell as an Hydrogen/Spyder/VScode script (#59)""" - default_comment_magics = False - parse_cell_language = False + default_comment_magics = True + parse_cell_language = True def code_to_text(self): """Not used""" @@ -299,6 +298,8 @@ def cell_to_text(self): self.metadata['cell_type'] = self.cell_type active = is_active('py', self.metadata) + if self.language != self.default_language and 'active' not in self.metadata: + active = False if self.cell_type == 'raw' and 'active' in self.metadata and self.metadata['active'] == '': del self.metadata['active'] @@ -308,11 +309,10 @@ def cell_to_text(self): else: lines = [self.comment + ' %% ' + options] - if self.cell_type == 'code': + if self.cell_type == 'code' and active: source = copy(self.source) comment_magic(source, self.language, self.comment_magics) - if active: - return lines + source + return lines + source return lines + comment_lines(self.source, self.comment) diff --git a/jupytext/cli.py b/jupytext/cli.py index b36718131..e2fce7d89 100644 --- a/jupytext/cli.py +++ b/jupytext/cli.py @@ -14,7 +14,7 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None, test_round_trip=False, test_round_trip_strict=False, stop_on_first_error=True, - update=True, freeze_metadata=False): + update=True, freeze_metadata=False, comment_magics=None): """ Export R markdown notebooks, python or R scripts, or Jupyter notebooks, to the opposite format @@ -26,6 +26,8 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None, :param test_round_trip_strict: should round trip conversion be tested, with strict notebook comparison? :param stop_on_first_error: when testing, should we stop on first error, or compare the full notebook? :param update: preserve the current outputs of .ipynb file + :param freeze_metadata: set metadata filters equal to the current script metadata + :param comment_magics: comment, or not, Jupyter magics when possible :return: """ @@ -81,6 +83,9 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None, print('{}: {}'.format(nb_file, str(error))) continue + if comment_magics is not None: + notebook.metadata.setdefault('jupytext', {})['comment_magics'] = comment_magics + if output == '-': sys.stdout.write(writes(notebook, ext=ext, format_name=format_name)) continue @@ -145,6 +150,18 @@ def canonize_format(format_or_ext, file_path=None): return {'notebook': 'ipynb', 'markdown': 'md', 'rmarkdown': 'Rmd'}[format_or_ext] +def str2bool(input): + """Parse Yes/No/Default string + https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse""" + if input.lower() in ('yes', 'true', 't', 'y', '1'): + return True + if input.lower() in ('no', 'false', 'f', 'n', '0'): + return False + if input.lower() in ('d', 'default', ''): + return None + raise argparse.ArgumentTypeError('Expected: (Y)es/(T)rue/(N)o/(F)alse/(D)efault') + + def cli_jupytext(args=None): """Command line parser for jupytext""" parser = argparse.ArgumentParser( @@ -177,6 +194,11 @@ def cli_jupytext(args=None): parser.add_argument('--update', action='store_true', help='Preserve outputs of .ipynb destination ' '(when file exists and inputs match)') + parser.add_argument('--comment-magics', + type=str2bool, + nargs='?', + default=None, + help='Should Jupyter magic commands be commented? (Y)es/(T)rue/(N)o/(F)alse/(D)efault') parser.add_argument('--freeze-metadata', action='store_true', help='Set a metadata filter (unless one exists already) ' 'equal to the current metadata of the notebook. Use this ' @@ -231,7 +253,8 @@ def jupytext(args=None): test_round_trip_strict=args.test_strict, stop_on_first_error=args.stop_on_first_error, update=args.update, - freeze_metadata=args.freeze_metadata) + freeze_metadata=args.freeze_metadata, + comment_magics=args.comment_magics) except ValueError as err: # (ValueError, TypeError, IOError) as err: print('jupytext: error: ' + str(err)) exit(1) diff --git a/jupytext/formats.py b/jupytext/formats.py index c3f6d65fb..7d68c558a 100644 --- a/jupytext/formats.py +++ b/jupytext/formats.py @@ -87,10 +87,12 @@ def __init__(self, header_prefix=_SCRIPT_EXTENSIONS[ext]['comment'], cell_reader_class=DoublePercentScriptCellReader, cell_exporter_class=DoublePercentCellExporter, + # Version 1.2 on 2018-11-18 - jupytext v0.8.6: Jupyter magics are commented by default #126, #132 # Version 1.1 on 2018-09-23 - jupytext v0.7.0rc1 : [markdown] and # [raw] for markdown and raw cells. # Version 1.0 on 2018-09-22 - jupytext v0.7.0rc0 : Initial version - current_version_number='1.1') for ext in _SCRIPT_EXTENSIONS] + \ + current_version_number='1.2', + min_readable_version_number='1.1') for ext in _SCRIPT_EXTENSIONS] + \ [ NotebookFormatDescription( format_name='sphinx', diff --git a/tests/notebooks/mirror/ipynb_to_percent/Notebook_with_R_magic.py b/tests/notebooks/mirror/ipynb_to_percent/Notebook_with_R_magic.py index 816bb977d..367549d0d 100644 --- a/tests/notebooks/mirror/ipynb_to_percent/Notebook_with_R_magic.py +++ b/tests/notebooks/mirror/ipynb_to_percent/Notebook_with_R_magic.py @@ -12,19 +12,16 @@ # This notebook shows the use of R cells to generate plots # %% -%load_ext rpy2.ipython +# %load_ext rpy2.ipython -# %% -%%R -suppressMessages(require(tidyverse)) +# %% {"language": "R"} +# suppressMessages(require(tidyverse)) -# %% -%%R -ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point() +# %% {"language": "R"} +# ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point() # %% [markdown] # The default plot dimensions are not good for us, so we use the -w and -h parameters in %%R magic to set the plot size -# %% -%%R -w 400 -h 240 -ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point() +# %% {"magic_args": "-w 400 -h 240", "language": "R"} +# ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point() diff --git a/tests/notebooks/mirror/ipynb_to_percent/convert_to_py_then_test_with_update83.py b/tests/notebooks/mirror/ipynb_to_percent/convert_to_py_then_test_with_update83.py index 8300b7c11..236ebdd61 100644 --- a/tests/notebooks/mirror/ipynb_to_percent/convert_to_py_then_test_with_update83.py +++ b/tests/notebooks/mirror/ipynb_to_percent/convert_to_py_then_test_with_update83.py @@ -10,7 +10,7 @@ # --- # %% -%%time +# %%time print('asdf') diff --git a/tests/notebooks/mirror/ipynb_to_percent/jupyter_again.py b/tests/notebooks/mirror/ipynb_to_percent/jupyter_again.py index 6cdee6b74..2b1a51ee5 100644 --- a/tests/notebooks/mirror/ipynb_to_percent/jupyter_again.py +++ b/tests/notebooks/mirror/ipynb_to_percent/jupyter_again.py @@ -22,4 +22,4 @@ print(yaml.dump(yaml.load(c))) # %% -?next +# ?next diff --git a/tests/notebooks/mirror/ipynb_to_percent/nteract_with_parameter.py b/tests/notebooks/mirror/ipynb_to_percent/nteract_with_parameter.py index 4f9ba3209..1bae8a36c 100644 --- a/tests/notebooks/mirror/ipynb_to_percent/nteract_with_parameter.py +++ b/tests/notebooks/mirror/ipynb_to_percent/nteract_with_parameter.py @@ -20,5 +20,5 @@ df # %% {"outputHidden": false, "inputHidden": false} -%matplotlib inline +# %matplotlib inline df.plot(kind='bar') diff --git a/tests/notebooks/mirror/ipynb_to_percent/xcpp_by_quantstack.cpp b/tests/notebooks/mirror/ipynb_to_percent/xcpp_by_quantstack.cpp index 249ccf6eb..7618df8bb 100644 --- a/tests/notebooks/mirror/ipynb_to_percent/xcpp_by_quantstack.cpp +++ b/tests/notebooks/mirror/ipynb_to_percent/xcpp_by_quantstack.cpp @@ -373,7 +373,7 @@ xcpp::display(rect, "some_display_id", true); std::vector to_shuffle = {1, 2, 3, 4}; // %% -%timeit std::random_shuffle(to_shuffle.begin(), to_shuffle.end()); +// %timeit std::random_shuffle(to_shuffle.begin(), to_shuffle.end()); // %% [markdown] // [![xtensor](images/xtensor.png)](https://github.com/QuantStack/xtensor/) diff --git a/tests/test_cli.py b/tests/test_cli.py index 8fc17e239..37ee0b945 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -223,3 +223,24 @@ def test_convert_to_percent_format(nb_file, tmpdir): nb2 = readf(tmp_nbpy) compare_notebooks(nb1, nb2) + + +@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_py')) +def test_convert_to_percent_format_and_keep_magics(nb_file, tmpdir): + tmp_ipynb = str(tmpdir.join('notebook.ipynb')) + tmp_nbpy = str(tmpdir.join('notebook.py')) + + copyfile(nb_file, tmp_ipynb) + + with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True): + jupytext(['--to', 'py:percent', '--comment-magics', 'no', tmp_ipynb]) + + with open(tmp_nbpy) as stream: + py_script = stream.read() + assert 'format_name: percent' in py_script + assert '# %%time' not in py_script + + nb1 = readf(tmp_ipynb) + nb2 = readf(tmp_nbpy) + + compare_notebooks(nb1, nb2) diff --git a/tests/test_escape_magics.py b/tests/test_escape_magics.py index f46ff4818..3b23c5c0f 100644 --- a/tests/test_escape_magics.py +++ b/tests/test_escape_magics.py @@ -36,7 +36,7 @@ def test_force_noescape_with_gbl_esc_flag(line): @pytest.mark.parametrize('ext_and_format_name,commented', zip(['md', 'Rmd', 'py:light', 'py:percent', 'py:sphinx', 'R', 'ss:light', 'ss:percent'], - [False, True, True, False, True, True, True, False])) + [False, True, True, True, True, True, True, True])) def test_magics_commented_default(ext_and_format_name, commented): ext, format_name = parse_one_format(ext_and_format_name) nb = new_notebook(cells=[new_code_cell('%pylab inline')]) @@ -99,9 +99,9 @@ def test_force_comment_using_contents_manager(tmpdir): cm.save(model=dict(type='notebook', content=nb), path=tmp_py) with open(str(tmpdir.join(tmp_py))) as stream: - assert '%pylab inline' in stream.read().splitlines() + assert '# %pylab inline' in stream.read().splitlines() - cm.comment_magics = True + cm.comment_magics = False cm.save(model=dict(type='notebook', content=nb), path=tmp_py) with open(str(tmpdir.join(tmp_py))) as stream: - assert '# %pylab inline' in stream.read().splitlines() + assert '%pylab inline' in stream.read().splitlines() diff --git a/tests/test_read_simple_percent.py b/tests/test_read_simple_percent.py index 8bd12a2c9..59244064b 100644 --- a/tests/test_read_simple_percent.py +++ b/tests/test_read_simple_percent.py @@ -48,7 +48,7 @@ def test_read_simple_file(script="""# --- compare(nb.cells[5].source, '''1 + 2 + 3 + 4 5 6 -# %%magic # this is a commented magic, not a cell +%%magic # this is a commented magic, not a cell 7''') assert nb.cells[5].metadata == {'title': 'And now a code cell'}