Percent format v1.2: magic commands are commented #126 #132

.travis.yml

@@ -19,7 +19,7 @@ install:
   - pip install .
 before_script:
   # stop the build if there are Python syntax errors or undefined names
-  - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --exclude=ipynb_to_percent
+  - flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
   # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
   - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 script:

HISTORY.rst

@@ -10,6 +10,8 @@ Release History
 
 - The ``language_info`` section is not part of the default header any more. Language information is now taken from metadata ``kernelspec.language``. (#105).
 - When opening a paired notebook, the active file is now the file that was originally opened (#118). When saving a notebook, timestamps of all the alternative representations are tested to ensure that Jupyter's autosave does not override manual modifications.
+- Jupyter magic commands are now commented per default in the ``percent`` format (#126, #132). Version for the ``percent`` format increases from '1.1' to '1.2'. Set an option ``comment_magics`` to ``false`` either per notebook, or globally on Jupytext's contents manager, or on `jupytext`'s command line, if you prefer not to comment Jupyter magics.
+
 
 0.8.5 (2018-11-13)
 ++++++++++++++++++++++

README.md

@@ -138,6 +138,7 @@ The package provides a `jupytext` script for command line conversion between the
 ```bash
 jupytext --to python notebook.ipynb             # create a notebook.py file
 jupytext --to py:percent notebook.ipynb         # create a notebook.py file in the double percent format
+jupytext --to py:percent --comment-magics false notebook.ipynb   # create a notebook.py file in the double percent format, and do not comment magic commands
 jupytext --to markdown notebook.ipynb           # create a notebook.md file
 jupytext --output script.py notebook.ipynb      # create a script.py file
 
@@ -282,7 +283,7 @@ That being said, Jupytext also works well from Jupyter Lab. Please note that:
 
 ## Will my notebook really run in an IDE?
 
-Well, that's what we expect. There's however a big difference in the python environments between Python IDEs and Jupyter: in most IDEs the code is executed with  `python` and not in a Jupyter kernel. For this reason, `jupytext` comments Jupyter magics found in your notebook when exporting to R Markdown, and to scripts in all format but the `percent` one. Magics are not commented in the plain Markdown representation, nor in the `percent` format, as some editors use that format in combination with Jupyter kernels. Change this by adding a `#escape` or `#noescape` flag on the same line as the magic, or a `"comment_magics": true` or `false` entry in the notebook metadata, in the `"jupytext"` section. Or set your preference globally on the contents manager by adding this line to `.jupyter/jupyter_notebook_config.py`:
+Well, that's what we expect. There's however a big difference in the python environments between Python IDEs and Jupyter: in most IDEs the code is executed with  `python` and not in a Jupyter kernel. For this reason, `jupytext` comments Jupyter magics found in your notebook when exporting to all format but the plain Markdown one. Change this by adding a `#escape` or `#noescape` flag on the same line as the magic, or a `"comment_magics": true` or `false` entry in the notebook metadata, in the `"jupytext"` section. Or set your preference globally on the contents manager by adding this line to `.jupyter/jupyter_notebook_config.py`:
 ```python
 c.ContentsManager.comment_magics = True # or False
 ```

jupytext/cell_reader.py

@@ -408,7 +408,7 @@ def find_cell_end(self, lines):
 
 class DoublePercentScriptCellReader(ScriptCellReader):
     """Read notebook cells from Hydrogen/Spyder/VScode scripts (#59)"""
-    default_comment_magics = False
+    default_comment_magics = True
 
     def __init__(self, ext, comment_magics=None):
         ScriptCellReader.__init__(self, ext, comment_magics)
@@ -443,9 +443,10 @@ def find_cell_content(self, lines):
         # Cell content
         source = lines[cell_start:cell_end_marker]
 
-        if self.cell_type != 'code' or (self.metadata and not is_active('py', self.metadata)):
+        if self.cell_type != 'code' or (self.metadata and not is_active('py', self.metadata)) \
+                or (self.language is not None and self.language != self.default_language):
             source = uncomment(source, self.comment)
-        if self.comment_magics:
+        elif self.metadata is not None and self.comment_magics:
             source = self.uncomment_code_and_magics(source)
 
         self.content = source

jupytext/cell_to_text.py

@@ -60,8 +60,7 @@ def __init__(self, cell, default_language, ext, comment_magics=None, cell_metada
 
         # how many blank lines before next cell
         self.lines_to_next_cell = cell.metadata.get('lines_to_next_cell', 1)
-        self.lines_to_end_of_cell_marker = \
-            cell.metadata.get('lines_to_end_of_cell_marker', 0)
+        self.lines_to_end_of_cell_marker = cell.metadata.get('lines_to_end_of_cell_marker', 0)
 
         # for compatibility with v0.5.4 and lower (to be removed)
         if 'skipline' in cell.metadata:
@@ -286,8 +285,8 @@ def code_to_text(self):
 class DoublePercentCellExporter(BaseCellExporter):
     """A class that can represent a notebook cell as an
     Hydrogen/Spyder/VScode script (#59)"""
-    default_comment_magics = False
-    parse_cell_language = False
+    default_comment_magics = True
+    parse_cell_language = True
 
     def code_to_text(self):
         """Not used"""
@@ -299,6 +298,8 @@ def cell_to_text(self):
             self.metadata['cell_type'] = self.cell_type
 
         active = is_active('py', self.metadata)
+        if self.language != self.default_language and 'active' not in self.metadata:
+            active = False
         if self.cell_type == 'raw' and 'active' in self.metadata and self.metadata['active'] == '':
             del self.metadata['active']
 
@@ -308,11 +309,10 @@ def cell_to_text(self):
         else:
             lines = [self.comment + ' %% ' + options]
 
-        if self.cell_type == 'code':
+        if self.cell_type == 'code' and active:
             source = copy(self.source)
             comment_magic(source, self.language, self.comment_magics)
-            if active:
-                return lines + source
+            return lines + source
 
         return lines + comment_lines(self.source, self.comment)
 

jupytext/cli.py

@@ -14,7 +14,7 @@
 
 def convert_notebook_files(nb_files, fmt, input_format=None, output=None,
                            test_round_trip=False, test_round_trip_strict=False, stop_on_first_error=True,
-                           update=True, freeze_metadata=False):
+                           update=True, freeze_metadata=False, comment_magics=None):
     """
     Export R markdown notebooks, python or R scripts, or Jupyter notebooks,
     to the opposite format
@@ -26,6 +26,8 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None,
     :param test_round_trip_strict: should round trip conversion be tested, with strict notebook comparison?
     :param stop_on_first_error: when testing, should we stop on first error, or compare the full notebook?
     :param update: preserve the current outputs of .ipynb file
+    :param freeze_metadata: set metadata filters equal to the current script metadata
+    :param comment_magics: comment, or not, Jupyter magics
     when possible
     :return:
     """
@@ -81,6 +83,9 @@ def convert_notebook_files(nb_files, fmt, input_format=None, output=None,
                 print('{}: {}'.format(nb_file, str(error)))
             continue
 
+        if comment_magics is not None:
+            notebook.metadata.setdefault('jupytext', {})['comment_magics'] = comment_magics
+
         if output == '-':
             sys.stdout.write(writes(notebook, ext=ext, format_name=format_name))
             continue
@@ -145,6 +150,18 @@ def canonize_format(format_or_ext, file_path=None):
     return {'notebook': 'ipynb', 'markdown': 'md', 'rmarkdown': 'Rmd'}[format_or_ext]
 
 
+def str2bool(input):
+    """Parse Yes/No/Default string
+    https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse"""
+    if input.lower() in ('yes', 'true', 't', 'y', '1'):
+        return True
+    if input.lower() in ('no', 'false', 'f', 'n', '0'):
+        return False
+    if input.lower() in ('d', 'default', ''):
+        return None
+    raise argparse.ArgumentTypeError('Expected: (Y)es/(T)rue/(N)o/(F)alse/(D)efault')
+
+
 def cli_jupytext(args=None):
     """Command line parser for jupytext"""
     parser = argparse.ArgumentParser(
@@ -177,6 +194,11 @@ def cli_jupytext(args=None):
     parser.add_argument('--update', action='store_true',
                         help='Preserve outputs of .ipynb destination '
                              '(when file exists and inputs match)')
+    parser.add_argument('--comment-magics',
+                        type=str2bool,
+                        nargs='?',
+                        default=None,
+                        help='Should Jupyter magic commands be commented? (Y)es/(T)rue/(N)o/(F)alse/(D)efault')
     parser.add_argument('--freeze-metadata', action='store_true',
                         help='Set a metadata filter (unless one exists already) '
                              'equal to the current metadata of the notebook. Use this '
@@ -231,7 +253,8 @@ def jupytext(args=None):
                                test_round_trip_strict=args.test_strict,
                                stop_on_first_error=args.stop_on_first_error,
                                update=args.update,
-                               freeze_metadata=args.freeze_metadata)
+                               freeze_metadata=args.freeze_metadata,
+                               comment_magics=args.comment_magics)
     except ValueError as err:  # (ValueError, TypeError, IOError) as err:
         print('jupytext: error: ' + str(err))
         exit(1)

jupytext/formats.py

@@ -87,10 +87,12 @@ def __init__(self,
             header_prefix=_SCRIPT_EXTENSIONS[ext]['comment'],
             cell_reader_class=DoublePercentScriptCellReader,
             cell_exporter_class=DoublePercentCellExporter,
+            # Version 1.2 on 2018-11-18 - jupytext v0.8.6: Jupyter magics are commented by default #126, #132
             # Version 1.1 on 2018-09-23 - jupytext v0.7.0rc1 : [markdown] and
             # [raw] for markdown and raw cells.
             # Version 1.0 on 2018-09-22 - jupytext v0.7.0rc0 : Initial version
-            current_version_number='1.1') for ext in _SCRIPT_EXTENSIONS] + \
+            current_version_number='1.2',
+            min_readable_version_number='1.1') for ext in _SCRIPT_EXTENSIONS] + \
     [
         NotebookFormatDescription(
             format_name='sphinx',

tests/notebooks/mirror/ipynb_to_percent/Notebook_with_R_magic.py

@@ -12,19 +12,16 @@
 # This notebook shows the use of R cells to generate plots
 
 # %%
-%load_ext rpy2.ipython
+# %load_ext rpy2.ipython
 
-# %%
-%%R
-suppressMessages(require(tidyverse))
+# %% {"language": "R"}
+# suppressMessages(require(tidyverse))
 
-# %%
-%%R
-ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point()
+# %% {"language": "R"}
+# ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point()
 
 # %% [markdown]
 # The default plot dimensions are not good for us, so we use the -w and -h parameters in %%R magic to set the plot size
 
-# %%
-%%R -w 400 -h 240
-ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point()
+# %% {"magic_args": "-w 400 -h 240", "language": "R"}
+# ggplot(iris, aes(x = Sepal.Length, y = Petal.Length, color=Species)) + geom_point()

tests/notebooks/mirror/ipynb_to_percent/convert_to_py_then_test_with_update83.py

@@ -10,7 +10,7 @@
 # ---
 
 # %%
-%%time
+# %%time
 
 print('asdf')
 

tests/notebooks/mirror/ipynb_to_percent/jupyter_again.py

@@ -22,4 +22,4 @@
 print(yaml.dump(yaml.load(c)))
 
 # %%
-?next
+# ?next

tests/notebooks/mirror/ipynb_to_percent/nteract_with_parameter.py

@@ -20,5 +20,5 @@
 df
 
 # %% {"outputHidden": false, "inputHidden": false}
-%matplotlib inline
+# %matplotlib inline
 df.plot(kind='bar')

tests/notebooks/mirror/ipynb_to_percent/xcpp_by_quantstack.cpp

@@ -373,7 +373,7 @@ xcpp::display(rect, "some_display_id", true);
 std::vector<double> to_shuffle = {1, 2, 3, 4};
 
 // %%
-%timeit std::random_shuffle(to_shuffle.begin(), to_shuffle.end());
+// %timeit std::random_shuffle(to_shuffle.begin(), to_shuffle.end());
 
 // %% [markdown]
 // [![xtensor](images/xtensor.png)](https://github.com/QuantStack/xtensor/)

tests/test_cli.py

@@ -223,3 +223,24 @@ def test_convert_to_percent_format(nb_file, tmpdir):
     nb2 = readf(tmp_nbpy)
 
     compare_notebooks(nb1, nb2)
+
+
+@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_py'))
+def test_convert_to_percent_format_and_keep_magics(nb_file, tmpdir):
+    tmp_ipynb = str(tmpdir.join('notebook.ipynb'))
+    tmp_nbpy = str(tmpdir.join('notebook.py'))
+
+    copyfile(nb_file, tmp_ipynb)
+
+    with mock.patch('jupytext.header.INSERT_AND_CHECK_VERSION_NUMBER', True):
+        jupytext(['--to', 'py:percent', '--comment-magics', 'no', tmp_ipynb])
+
+    with open(tmp_nbpy) as stream:
+        py_script = stream.read()
+        assert 'format_name: percent' in py_script
+        assert '# %%time' not in py_script
+
+    nb1 = readf(tmp_ipynb)
+    nb2 = readf(tmp_nbpy)
+
+    compare_notebooks(nb1, nb2)

tests/test_escape_magics.py

@@ -36,7 +36,7 @@ def test_force_noescape_with_gbl_esc_flag(line):
 
 @pytest.mark.parametrize('ext_and_format_name,commented',
                          zip(['md', 'Rmd', 'py:light', 'py:percent', 'py:sphinx', 'R', 'ss:light', 'ss:percent'],
-                             [False, True, True, False, True, True, True, False]))
+                             [False, True, True, True, True, True, True, True]))
 def test_magics_commented_default(ext_and_format_name, commented):
     ext, format_name = parse_one_format(ext_and_format_name)
     nb = new_notebook(cells=[new_code_cell('%pylab inline')])
@@ -99,9 +99,9 @@ def test_force_comment_using_contents_manager(tmpdir):
 
     cm.save(model=dict(type='notebook', content=nb), path=tmp_py)
     with open(str(tmpdir.join(tmp_py))) as stream:
-        assert '%pylab inline' in stream.read().splitlines()
+        assert '# %pylab inline' in stream.read().splitlines()
 
-    cm.comment_magics = True
+    cm.comment_magics = False
     cm.save(model=dict(type='notebook', content=nb), path=tmp_py)
     with open(str(tmpdir.join(tmp_py))) as stream:
-        assert '# %pylab inline' in stream.read().splitlines()
+        assert '%pylab inline' in stream.read().splitlines()

tests/test_read_simple_percent.py

@@ -48,7 +48,7 @@ def test_read_simple_file(script="""# ---
     compare(nb.cells[5].source, '''1 + 2 + 3 + 4
 5
 6
-# %%magic # this is a commented magic, not a cell
+%%magic # this is a commented magic, not a cell
 
 7''')
     assert nb.cells[5].metadata == {'title': 'And now a code cell'}