From 76e6f29d40e3b912fb2d607fc2ec11ac0b048a97 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Fri, 22 May 2020 16:55:03 +0200 Subject: [PATCH 1/2] Pair notebooks in trees with /// --- docs/config.md | 6 +++--- jupytext/paired_paths.py | 34 ++++++++++++++++++++++++++++++++++ tests/test_cli.py | 32 ++++++++++++++++++++++++++++++++ tests/test_paired_paths.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/docs/config.md b/docs/config.md index deaab4052..58c41778b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -6,14 +6,14 @@ jupytext --set-formats ipynb,py [--sync] notebook.ipynb ``` You can pair a notebook to as many text representations as you want (see our _World population_ notebook in the demo folder). Format specifications are of the form ``` -[[path/][prefix]/][suffix.]ext[:format_name] +[[root_folder//][path/][prefix]/][suffix.]ext[:format_name] ``` where - `ext` is one of `ipynb`, `md`, `Rmd`, `jl`, `py`, `R`, `sh`, `cpp`, `q`. Use the `auto` extension to have the script extension chosen according to the Jupyter kernel. - `format_name` (optional) is either `light` (default for scripts), `nomarker`, `percent`, `hydrogen`, `sphinx` (Python only), `spin` (R only) — see the [format specifications](formats.md). -- `path`, `prefix` and `suffix` allow to save the text representation to files with different names, or in a different folder. +- `root_folder`, `path`, `prefix` and `suffix` allow to save the text representation to files with different names, or in a different folder. -If you want to pair a notebook to a python script in a subfolder named `scripts`, set the formats metadata to `ipynb,scripts//py`. If the notebook is in a `notebooks` folder and you want the text representation to be in a `scripts` folder at the same level, set the Jupytext formats to `notebooks//ipynb,scripts//py`. +If you want to pair a notebook to a python script in a subfolder named `scripts`, set the formats metadata to `ipynb,scripts//py`. If the notebook is in a `notebooks` folder and you want the text representation to be in a `scripts` folder at the same level, set the Jupytext formats to `notebooks//ipynb,scripts//py`. If you want to pair the notebooks in subtrees, use e.g. `notebooks///ipynb,scripts///py` (and make sure you don't use `notebooks` and trees in subfolder names). Jupytext accepts a few additional options. These options should be added to the `"jupytext"` section in the metadata — use either the metadata editor or the `--opt/--format-options` argument on the command line. - `comment_magics`: By default, Jupyter magics are commented when notebooks are exported to any other format than markdown. If you prefer otherwise, use this boolean option, or is global counterpart (see below). diff --git a/jupytext/paired_paths.py b/jupytext/paired_paths.py index 905687b13..54c93e96f 100644 --- a/jupytext/paired_paths.py +++ b/jupytext/paired_paths.py @@ -48,6 +48,10 @@ def base_path(main_path, fmt): if not prefix: return base + if "//" in prefix: + prefix_root, prefix = prefix.rsplit("//", 1) + else: + prefix_root = "" prefix_dir, prefix_file_name = os.path.split(prefix) notebook_dir, notebook_file_name = os.path.split(base) sep = base[len(notebook_dir) : -len(notebook_file_name)] @@ -89,6 +93,18 @@ def base_path(main_path, fmt): ) notebook_dir = parent_notebook_dir + if prefix_root: + long_prefix_root = sep + prefix_root + sep + long_notebook_dir = sep + notebook_dir + sep + if long_prefix_root not in long_notebook_dir: + raise InconsistentPath( + u"Notebook directory '{}' does not match prefix root '{}'".format( + notebook_dir, prefix_root + ) + ) + notebook_dir = "///".join(long_notebook_dir.rsplit(long_prefix_root, 1)) + notebook_dir = notebook_dir[len(sep) : -len(sep)] + if not notebook_dir: return notebook_file_name @@ -108,6 +124,10 @@ def full_path(base, fmt): full = base if prefix: + if "//" in prefix: + prefix_root, prefix = prefix.rsplit("//", 1) + else: + prefix_root = "" prefix_dir, prefix_file_name = os.path.split(prefix) notebook_dir, notebook_file_name = os.path.split(base) @@ -115,6 +135,20 @@ def full_path(base, fmt): sep = base[len(notebook_dir) : -len(notebook_file_name)] or "/" prefix_dir = prefix_dir.replace("/", sep) + if (prefix_root != "") != ("//" in notebook_dir): + raise InconsistentPath( + u"Notebook base name '{}' is not compatible with fmt={}. Make sure you use prefix roots " + u"in either none, or all of the paired formats".format( + base, short_form_one_format(fmt) + ) + ) + + if prefix_root: + long_prefix_root = prefix_root + sep + long_notebook_dir = sep + notebook_dir + sep + long_notebook_dir = long_prefix_root.join(long_notebook_dir.rsplit("//", 1)) + notebook_dir = long_notebook_dir[len(sep) : -len(sep)] + if prefix_file_name: notebook_file_name = prefix_file_name + notebook_file_name diff --git a/tests/test_cli.py b/tests/test_cli.py index d2c2a04c6..3adb6a075 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1013,3 +1013,35 @@ def test_set_option_split_at_heading(tmpdir): cells=[new_markdown_cell("A paragraph"), new_markdown_cell("# H1 Header")] ) compare_notebooks(nb, nb_expected) + + +def test_pair_in_tree(tmpdir): + nb_file = tmpdir.mkdir("notebooks").mkdir("subfolder").join("example.ipynb") + py_file = tmpdir.mkdir("scripts").mkdir("subfolder").join("example.py") + + write(new_notebook(cells=[new_markdown_cell("A markdown cell")]), str(nb_file)) + + jupytext(["--set-formats", "notebooks///ipynb,scripts///py:percent", str(nb_file)]) + + assert py_file.exists() + assert "A markdown cell" in py_file.read() + + +def test_pair_in_tree_and_parent(tmpdir): + nb_file = ( + tmpdir.mkdir("notebooks") + .mkdir("subfolder") + .mkdir("a") + .mkdir("b") + .join("example.ipynb") + ) + py_file = tmpdir.mkdir("scripts").mkdir("subfolder").mkdir("c").join("example.py") + + write(new_notebook(cells=[new_markdown_cell("A markdown cell")]), str(nb_file)) + + jupytext( + ["--set-formats", "notebooks//a/b//ipynb,scripts//c//py:percent", str(nb_file)] + ) + + assert py_file.exists() + assert "A markdown cell" in py_file.read() diff --git a/tests/test_paired_paths.py b/tests/test_paired_paths.py index 7f9905e65..54b20055c 100644 --- a/tests/test_paired_paths.py +++ b/tests/test_paired_paths.py @@ -38,6 +38,34 @@ def test_full_path_dotdot(): assert full_path("scripts/test", fmt=fmt) == "scripts/test.py" +def test_base_path_in_tree_from_root(): + fmt = long_form_one_format("scripts///py") + assert base_path("scripts/subfolder/test.py", fmt=fmt) == "//subfolder/test" + assert base_path("/scripts/subfolder/test.py", fmt=fmt) == "///subfolder/test" + + +def test_base_path_in_tree_from_non_root(): + fmt = long_form_one_format("scripts///py") + assert ( + base_path("/parent_folder/scripts/subfolder/test.py", fmt=fmt) + == "/parent_folder///subfolder/test" + ) + + +def test_full_path_in_tree_from_root(): + fmt = long_form_one_format("notebooks///ipynb") + assert full_path("//subfolder/test", fmt=fmt) == "notebooks/subfolder/test.ipynb" + assert full_path("///subfolder/test", fmt=fmt) == "/notebooks/subfolder/test.ipynb" + + +def test_full_path_in_tree_from_non_root(): + fmt = long_form_one_format("notebooks///ipynb") + assert ( + full_path("/parent_folder///subfolder/test", fmt=fmt) + == "/parent_folder/notebooks/subfolder/test.ipynb" + ) + + def test_many_and_suffix(): formats = long_form_multiple_formats("ipynb,.pct.py,_lgt.py") expected_paths = ["notebook.ipynb", "notebook.pct.py", "notebook_lgt.py"] From 18e0e44075558f58badb5e7eab746b3d03243ce6 Mon Sep 17 00:00:00 2001 From: Marc Wouts Date: Fri, 22 May 2020 16:58:15 +0200 Subject: [PATCH 2/2] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 767a0cecf..27ba4ee96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ **Added** - Jupytext can use a local or global [configuration file](https://github.com/mwouts/jupytext/blob/master/docs/config.md) (#508) +- Jupytext can pair notebooks in trees. Use e.g. `notebooks///ipynb,scripts///py:percent` if you want to replicate the arborescence of notebooks under `notebooks` in a folder named `scripts` (#424) - Jupytext is tested in `pip` and `conda` environments, on Linux, Mac OS and Windows, using Github actions (#487) - Pre-commit checks and automatic reformatting of Jupytext's code with `pre-commit`, `black` and `flake8` (#483) - Groovy and Java are now supported, thanks to Przemek WesoĊ‚ek's contribution (#500)