Jupytext CLI rewritten

Many new arguments: pipe, exec, quiet. #154 #142

jupytext/formats.py

@@ -149,7 +149,7 @@ def get_format_implementation(ext, format_name=None):
     if formats_for_extension:
         raise TypeError("Format '{}' is not associated to extension '{}'. "
                         "Please choose one of: {}.".format(format_name, ext, ', '.join(formats_for_extension)))
-    raise TypeError("Not format associated to extension '{}'".format(ext))
+    raise TypeError("No format associated to extension '{}'".format(ext))
 
 
 def read_metadata(text, ext):
@@ -285,13 +285,11 @@ def check_file_version(notebook, source_path, outputs_path):
     if (fmt.min_readable_version_number or current) <= version <= current:
         return
 
-    raise ValueError("File {} has jupytext_format_version={}, but "
-                     "current version for that extension is {}.\n"
-                     "It would not be safe to override the source of {} "
-                     "with that file.\n"
+    raise ValueError("File {} is in format/version={}/{} (current version is {}). "
+                     "It would not be safe to override the source of {} with that file. "
                      "Please remove one or the other file."
                      .format(os.path.basename(source_path),
-                             version, current,
+                             format_name, version, current,
                              os.path.basename(outputs_path)))
 
 
@@ -379,14 +377,21 @@ def rearrange_jupytext_metadata(metadata):
         metadata['jupytext'] = jupytext_metadata
 
 
-def long_form_one_format(jupytext_format):
+def long_form_one_format(jupytext_format, metadata=None):
     """Parse 'sfx.py:percent' into {'suffix':'sfx', 'extension':'py', 'format_name':'percent'}"""
     if isinstance(jupytext_format, dict):
         return jupytext_format
 
     if not jupytext_format:
         return {}
 
+    common_name_to_ext = {'notebook': 'ipynb',
+                          'rmarkdown': 'Rmd',
+                          'markdown': 'md',
+                          'c++': 'cpp'}
+    if jupytext_format.lower() in common_name_to_ext:
+        jupytext_format = common_name_to_ext[jupytext_format.lower()]
+
     fmt = {}
 
     if jupytext_format.rfind('/') > 0:
@@ -403,6 +408,9 @@ def long_form_one_format(jupytext_format):
     if not ext.startswith('.'):
         ext = '.' + ext
 
+    if ext == '.auto' and metadata:
+        ext = auto_ext_from_metadata(metadata)
+
     fmt['extension'] = ext
     return fmt
 

jupytext/jupytext.py

@@ -2,14 +2,15 @@
 
 import os
 import io
+import sys
 from copy import copy, deepcopy
 from nbformat.v4.rwbase import NotebookReader, NotebookWriter
 from nbformat.v4.nbbase import new_notebook, new_code_cell
 import nbformat
-from .formats import get_format_implementation, read_format_from_metadata, guess_format, long_form_one_format, \
-    update_jupytext_formats_metadata, format_name_for_ext, rearrange_jupytext_metadata
-from .header import header_to_metadata_and_cell, metadata_and_cell_to_header, \
-    encoding_and_executable, insert_or_test_version_number
+from .formats import read_format_from_metadata, update_jupytext_formats_metadata, rearrange_jupytext_metadata
+from .formats import format_name_for_ext, guess_format, divine_format, get_format_implementation, long_form_one_format
+from .header import header_to_metadata_and_cell, metadata_and_cell_to_header
+from .header import encoding_and_executable, insert_or_test_version_number
 from .metadata_filter import update_metadata_filters
 from .languages import default_language_from_metadata_and_ext, set_main_and_cell_language
 from .pep8 import pep8_lines_between_cells
@@ -167,6 +168,7 @@ def reads(text, fmt, as_version=4, **kwargs):
 
 def read(file_or_stream, fmt, as_version=4, **kwargs):
     """Read a notebook from a file"""
+    fmt = long_form_one_format(fmt)
     if fmt['extension'] == '.ipynb':
         notebook = nbformat.read(file_or_stream, as_version, **kwargs)
         rearrange_jupytext_metadata(notebook.metadata)
@@ -177,6 +179,11 @@ def read(file_or_stream, fmt, as_version=4, **kwargs):
 
 def readf(nb_file, fmt=None):
     """Read a notebook from the file with given name"""
+    if nb_file == '-':
+        text = sys.stdin.read()
+        fmt = fmt or divine_format(text)
+        return reads(text, fmt)
+
     _, ext = os.path.splitext(nb_file)
     fmt = copy(fmt or {})
     fmt.update({'extension': ext})
@@ -188,7 +195,7 @@ def writes(notebook, fmt, version=nbformat.NO_CONVERT, **kwargs):
     """Write a notebook to a string"""
     rearrange_jupytext_metadata(notebook.metadata)
     fmt = copy(fmt)
-    fmt = long_form_one_format(fmt)
+    fmt = long_form_one_format(fmt, notebook.metadata)
     ext = fmt['extension']
     format_name = fmt.get('format_name')
 
@@ -219,6 +226,10 @@ def write(notebook, file_or_stream, fmt, version=nbformat.NO_CONVERT, **kwargs):
 
 def writef(notebook, nb_file, fmt=None):
     """Write a notebook to the file with given name"""
+    if nb_file == '-':
+        write(notebook, sys.stdout, fmt)
+        return
+
     _, ext = os.path.splitext(nb_file)
     fmt = copy(fmt or {})
     fmt = long_form_one_format(fmt)

jupytext/paired_paths.py

@@ -1,7 +1,7 @@
 """List all the paths associated to a given notebook"""
 
 import os
-from .formats import long_form_multiple_formats
+from .formats import long_form_one_format, long_form_multiple_formats
 
 
 class InconsistentPath(ValueError):
@@ -12,6 +12,10 @@ class InconsistentPath(ValueError):
 
 def base_path(main_path, fmt):
     """Given a path and options for a format (ext, suffix, prefix), return the corresponding base path"""
+    if not fmt:
+        return os.path.splitext(main_path)[0]
+
+    fmt = long_form_one_format(fmt)
     fmt_ext = fmt['extension']
     suffix = fmt.get('suffix')
     prefix = fmt.get('prefix')
@@ -54,11 +58,11 @@ def base_path(main_path, fmt):
     return notebook_dir + sep + notebook_file_name
 
 
-def full_path(base, format_options):
+def full_path(base, fmt):
     """Return the full path for the notebook, given the base path"""
-    ext = format_options['extension']
-    suffix = format_options.get('suffix')
-    prefix = format_options.get('prefix')
+    ext = fmt['extension']
+    suffix = fmt.get('suffix')
+    prefix = fmt.get('prefix')
 
     full = base
 

setup.py

@@ -23,7 +23,7 @@
     long_description_content_type='text/markdown',
     url='https://github.com/mwouts/jupytext',
     packages=find_packages(),
-    entry_points={'console_scripts': ['jupytext = jupytext.cli:jupytext']},
+    entry_points={'console_scripts': ['jupytext = jupytext.cli:jupytext_cli']},
     tests_require=['pytest'],
     install_requires=['nbformat>=4.0.0', 'mock', 'pyyaml', 'testfixtures'],
     license='MIT',

tests/test_black.py

@@ -1,20 +1,24 @@
+import os
 import pytest
 from shutil import copyfile
 from testfixtures import compare
+from nbformat.v4.nbbase import new_notebook, new_code_cell
 from .utils import list_notebooks
-from jupytext import readf
-from jupytext.cli import system, jupytext
+from jupytext import readf, writef
+from jupytext.cli import system, jupytext, pipe_notebook
 from jupytext.combine import black_invariant
 
 
-def black_version():
+def tool_version(tool):
     try:
-        return system('black', '--version')
+        return system(tool, '--version')
     except OSError:  # pragma: no cover
         return None
 
 
-requires_black = pytest.mark.skipif(not black_version(), reason='black not found')
+requires_black = pytest.mark.skipif(not tool_version('black'), reason='black not found')
+requires_flake8 = pytest.mark.skipif(not tool_version('flake8'), reason='flake8 not found')
+requires_autopep8 = pytest.mark.skipif(not tool_version('autopep8'), reason='autopep8 not found')
 
 
 @requires_black
@@ -43,3 +47,74 @@ def test_apply_black_on_python_notebooks(nb_file, tmpdir):
             compare(c1.outputs, c2.outputs)
 
     compare(nb1.metadata, nb2.metadata)
+
+
+@requires_black
+def test_pipe_into_black():
+    nb_org = new_notebook(cells=[new_code_cell('1        +1')])
+    nb_dest = new_notebook(cells=[new_code_cell('1 + 1')])
+
+    nb_pipe = pipe_notebook(nb_org, 'black -')
+    compare(nb_dest, nb_pipe)
+
+
+@requires_autopep8
+def test_pipe_into_autopep8():
+    nb_org = new_notebook(cells=[new_code_cell('1        +1')])
+    nb_dest = new_notebook(cells=[new_code_cell('1 + 1')])
+
+    nb_pipe = pipe_notebook(nb_org, 'autopep8 -')
+    compare(nb_dest, nb_pipe)
+
+
+@requires_flake8
+def test_pipe_into_flake8():
+    # Notebook OK
+    nb = new_notebook(cells=[new_code_cell('# correct code\n1 + 1')])
+    pipe_notebook(nb, 'flake8 -', update=False)
+
+    # Notebook not OK
+    nb = new_notebook(cells=[new_code_cell('incorrect code')])
+    with pytest.raises(SystemExit):
+        pipe_notebook(nb, 'flake8 -', update=False)
+
+
+@requires_black
+@pytest.mark.parametrize('nb_file', list_notebooks('ipynb_py')[:1])
+def test_apply_black_through_jupytext(tmpdir, nb_file):
+    # Load real notebook metadata to get the 'auto' extension in --pipe-fmt to work
+    metadata = readf(nb_file).metadata
+
+    nb_org = new_notebook(cells=[new_code_cell('1        +1')], metadata=metadata)
+    nb_black = new_notebook(cells=[new_code_cell('1 + 1')], metadata=metadata)
+
+    os.makedirs(str(tmpdir.join('notebook_folder')))
+    os.makedirs(str(tmpdir.join('script_folder')))
+
+    tmp_ipynb = str(tmpdir.join('notebook_folder').join('notebook.ipynb'))
+    tmp_py = str(tmpdir.join('script_folder').join('notebook.py'))
+
+    # Black in place
+    writef(nb_org, tmp_ipynb)
+    jupytext([tmp_ipynb, '--pipe', 'black -'])
+    nb_now = readf(tmp_ipynb)
+    compare(nb_black, nb_now)
+
+    # Write to another folder using dots
+    script_fmt = os.path.join('..', 'script_folder//py:percent')
+    writef(nb_org, tmp_ipynb)
+    jupytext([tmp_ipynb, '--to', script_fmt, '--pipe', 'black -'])
+    assert os.path.isfile(tmp_py)
+    nb_now = readf(tmp_py)
+    nb_now.metadata = metadata
+    compare(nb_black, nb_now)
+    os.remove(tmp_py)
+
+    # Map to another folder based on file name
+    writef(nb_org, tmp_ipynb)
+    jupytext([tmp_ipynb, '--from', 'notebook_folder//ipynb', '--to', 'script_folder//py:percent',
+              '--pipe', 'black -', '--exec', 'flake8 -'])
+    assert os.path.isfile(tmp_py)
+    nb_now = readf(tmp_py)
+    nb_now.metadata = metadata
+    compare(nb_black, nb_now)