Pair notebooks in trees

CHANGELOG.md

@@ -3,6 +3,7 @@
 
 **Added**
 - Jupytext can use a local or global [configuration file](https://github.com/mwouts/jupytext/blob/master/docs/config.md) (#508)
+- Jupytext can pair notebooks in trees. Use e.g. `notebooks///ipynb,scripts///py:percent` if you want to replicate the arborescence of notebooks under `notebooks` in a folder named `scripts` (#424)
 - Jupytext is tested in `pip` and `conda` environments, on Linux, Mac OS and Windows, using Github actions (#487)
 - Pre-commit checks and automatic reformatting of Jupytext's code with `pre-commit`, `black` and `flake8` (#483)
 - Groovy and Java are now supported, thanks to Przemek Wesołek's contribution (#500)

docs/config.md

@@ -6,14 +6,14 @@ jupytext --set-formats ipynb,py [--sync] notebook.ipynb
 ```
 You can pair a notebook to as many text representations as you want (see our _World population_ notebook in the demo folder). Format specifications are of the form
 ```
-[[path/][prefix]/][suffix.]ext[:format_name]
+[[root_folder//][path/][prefix]/][suffix.]ext[:format_name]
 ```
 where
 - `ext` is one of `ipynb`, `md`, `Rmd`, `jl`, `py`, `R`, `sh`, `cpp`, `q`. Use the `auto` extension to have the script extension chosen according to the Jupyter kernel.
 - `format_name` (optional) is either `light` (default for scripts), `nomarker`, `percent`, `hydrogen`, `sphinx` (Python only), `spin` (R only) — see the [format specifications](formats.md).
-- `path`, `prefix` and `suffix` allow to save the text representation to files with different names, or in a different folder.
+- `root_folder`, `path`, `prefix` and `suffix` allow to save the text representation to files with different names, or in a different folder.
 
-If you want to pair a notebook to a python script in a subfolder named `scripts`, set the formats metadata to `ipynb,scripts//py`. If the notebook is in a `notebooks` folder and you want the text representation to be in a `scripts` folder at the same level, set the Jupytext formats to `notebooks//ipynb,scripts//py`.
+If you want to pair a notebook to a python script in a subfolder named `scripts`, set the formats metadata to `ipynb,scripts//py`. If the notebook is in a `notebooks` folder and you want the text representation to be in a `scripts` folder at the same level, set the Jupytext formats to `notebooks//ipynb,scripts//py`. If you want to pair the notebooks in subtrees, use e.g. `notebooks///ipynb,scripts///py` (and make sure you don't use `notebooks` and trees in subfolder names).
 
 Jupytext accepts a few additional options. These options should be added to the `"jupytext"` section in the metadata — use either the metadata editor or the `--opt/--format-options` argument on the command line.
 - `comment_magics`: By default, Jupyter magics are commented when notebooks are exported to any other format than markdown. If you prefer otherwise, use this boolean option, or is global counterpart (see below).

jupytext/paired_paths.py

@@ -48,6 +48,10 @@ def base_path(main_path, fmt):
     if not prefix:
         return base
 
+    if "//" in prefix:
+        prefix_root, prefix = prefix.rsplit("//", 1)
+    else:
+        prefix_root = ""
     prefix_dir, prefix_file_name = os.path.split(prefix)
     notebook_dir, notebook_file_name = os.path.split(base)
     sep = base[len(notebook_dir) : -len(notebook_file_name)]
@@ -89,6 +93,18 @@ def base_path(main_path, fmt):
                     )
         notebook_dir = parent_notebook_dir
 
+    if prefix_root:
+        long_prefix_root = sep + prefix_root + sep
+        long_notebook_dir = sep + notebook_dir + sep
+        if long_prefix_root not in long_notebook_dir:
+            raise InconsistentPath(
+                u"Notebook directory '{}' does not match prefix root '{}'".format(
+                    notebook_dir, prefix_root
+                )
+            )
+        notebook_dir = "///".join(long_notebook_dir.rsplit(long_prefix_root, 1))
+        notebook_dir = notebook_dir[len(sep) : -len(sep)]
+
     if not notebook_dir:
         return notebook_file_name
 
@@ -108,13 +124,31 @@ def full_path(base, fmt):
     full = base
 
     if prefix:
+        if "//" in prefix:
+            prefix_root, prefix = prefix.rsplit("//", 1)
+        else:
+            prefix_root = ""
         prefix_dir, prefix_file_name = os.path.split(prefix)
         notebook_dir, notebook_file_name = os.path.split(base)
 
         # Local path separator (\\ on windows)
         sep = base[len(notebook_dir) : -len(notebook_file_name)] or "/"
         prefix_dir = prefix_dir.replace("/", sep)
 
+        if (prefix_root != "") != ("//" in notebook_dir):
+            raise InconsistentPath(
+                u"Notebook base name '{}' is not compatible with fmt={}. Make sure you use prefix roots "
+                u"in either none, or all of the paired formats".format(
+                    base, short_form_one_format(fmt)
+                )
+            )
+
+        if prefix_root:
+            long_prefix_root = prefix_root + sep
+            long_notebook_dir = sep + notebook_dir + sep
+            long_notebook_dir = long_prefix_root.join(long_notebook_dir.rsplit("//", 1))
+            notebook_dir = long_notebook_dir[len(sep) : -len(sep)]
+
         if prefix_file_name:
             notebook_file_name = prefix_file_name + notebook_file_name
 

tests/test_cli.py

@@ -1013,3 +1013,35 @@ def test_set_option_split_at_heading(tmpdir):
         cells=[new_markdown_cell("A paragraph"), new_markdown_cell("# H1 Header")]
     )
     compare_notebooks(nb, nb_expected)
+
+
+def test_pair_in_tree(tmpdir):
+    nb_file = tmpdir.mkdir("notebooks").mkdir("subfolder").join("example.ipynb")
+    py_file = tmpdir.mkdir("scripts").mkdir("subfolder").join("example.py")
+
+    write(new_notebook(cells=[new_markdown_cell("A markdown cell")]), str(nb_file))
+
+    jupytext(["--set-formats", "notebooks///ipynb,scripts///py:percent", str(nb_file)])
+
+    assert py_file.exists()
+    assert "A markdown cell" in py_file.read()
+
+
+def test_pair_in_tree_and_parent(tmpdir):
+    nb_file = (
+        tmpdir.mkdir("notebooks")
+        .mkdir("subfolder")
+        .mkdir("a")
+        .mkdir("b")
+        .join("example.ipynb")
+    )
+    py_file = tmpdir.mkdir("scripts").mkdir("subfolder").mkdir("c").join("example.py")
+
+    write(new_notebook(cells=[new_markdown_cell("A markdown cell")]), str(nb_file))
+
+    jupytext(
+        ["--set-formats", "notebooks//a/b//ipynb,scripts//c//py:percent", str(nb_file)]
+    )
+
+    assert py_file.exists()
+    assert "A markdown cell" in py_file.read()

tests/test_paired_paths.py

@@ -38,6 +38,34 @@ def test_full_path_dotdot():
     assert full_path("scripts/test", fmt=fmt) == "scripts/test.py"
 
 
+def test_base_path_in_tree_from_root():
+    fmt = long_form_one_format("scripts///py")
+    assert base_path("scripts/subfolder/test.py", fmt=fmt) == "//subfolder/test"
+    assert base_path("/scripts/subfolder/test.py", fmt=fmt) == "///subfolder/test"
+
+
+def test_base_path_in_tree_from_non_root():
+    fmt = long_form_one_format("scripts///py")
+    assert (
+        base_path("/parent_folder/scripts/subfolder/test.py", fmt=fmt)
+        == "/parent_folder///subfolder/test"
+    )
+
+
+def test_full_path_in_tree_from_root():
+    fmt = long_form_one_format("notebooks///ipynb")
+    assert full_path("//subfolder/test", fmt=fmt) == "notebooks/subfolder/test.ipynb"
+    assert full_path("///subfolder/test", fmt=fmt) == "/notebooks/subfolder/test.ipynb"
+
+
+def test_full_path_in_tree_from_non_root():
+    fmt = long_form_one_format("notebooks///ipynb")
+    assert (
+        full_path("/parent_folder///subfolder/test", fmt=fmt)
+        == "/parent_folder/notebooks/subfolder/test.ipynb"
+    )
+
+
 def test_many_and_suffix():
     formats = long_form_multiple_formats("ipynb,.pct.py,_lgt.py")
     expected_paths = ["notebook.ipynb", "notebook.pct.py", "notebook_lgt.py"]