From 5d50424179d196cbb0d40a00bb979268a46bbd55 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 15 Mar 2020 22:19:15 +0000 Subject: [PATCH] Add `nb_to_myst` and round-trip conversion tests --- myst_nb/convert.py | 71 ++++++++++++++++++- .../basic.ipynb} | 4 ++ tests/roundtrip/basic.mystnb | 27 +++++++ tests/test_convert.py | 42 ++++------- 4 files changed, 114 insertions(+), 30 deletions(-) rename tests/{test_convert/test_basic.ipynb => roundtrip/basic.ipynb} (88%) create mode 100644 tests/roundtrip/basic.mystnb diff --git a/myst_nb/convert.py b/myst_nb/convert.py index 307732b8..f9ffcfe2 100644 --- a/myst_nb/convert.py +++ b/myst_nb/convert.py @@ -1,8 +1,13 @@ +""" +This module contains round-trip conversion between +myst formatted text documents and notebooks. +""" import json from typing import List, Union from docutils.parsers.rst.directives.misc import TestDirective import nbformat as nbf +import yaml from mistletoe.base_elements import SourceLines from mistletoe.parse_context import ParseContext, get_parse_context, set_parse_context @@ -12,8 +17,11 @@ from myst_parser.parse_directives import parse_directive_text +DEFAULT_DIRECTIVE = "nb-code" + + def myst_to_nb( - text: Union[str, List[str], SourceLines], directive: str = "nb-cell" + text: Union[str, List[str], SourceLines], directive: str = DEFAULT_DIRECTIVE ) -> nbf.NotebookNode: """Convert text written in the myst format to a notebook. @@ -84,7 +92,8 @@ def myst_to_nb( token = item.node # type: CodeFence # Note: we ignore anything after the directive on the first line - # TODO: could log warning about this: ``if token.arguments != ""``` + # this is reserved for the optional lexer name + # TODO: could log warning about if token.arguments != lexer name # we use the TestDirective here, since `parse_directive_text` # is setup to skip any option validation for this class @@ -124,3 +133,61 @@ def myst_to_nb( set_parse_context(original_context) return notebook + + +def from_nbnode(value): + """Recursively convert NotebookNode to dict.""" + if isinstance(value, nbf.NotebookNode): + return {k: from_nbnode(v) for k, v in value.items()} + return value + + +def nb_to_myst(nb: nbf.NotebookNode, directive: str = DEFAULT_DIRECTIVE): + string = "" + + metadata = from_nbnode(nb.metadata) + metadata["nbformat"] = nb.nbformat + metadata["nbformat_minor"] = nb.nbformat_minor + + # we add the pygments lexer as a directive argument, for use by syntax highlighters + pygments_lexer = metadata.get("language_info", {}).get("pygments_lexer", None) + + string += "---\n" + string += yaml.safe_dump(metadata) + string += "---\n" + + last_cell_md = False + for i, cell in enumerate(nb.cells): + + if cell.cell_type == "markdown": + metadata = from_nbnode(cell.metadata) + if metadata or last_cell_md: + if metadata: + string += "\n+++ {}\n".format(json.dumps(metadata)) + else: + string += "\n+++\n" + string += cell.source + if not cell.source.endswith("\n"): + string += "\n" + last_cell_md = True + + elif cell.cell_type == "code": + string += "```{{{}}}".format(directive) + if pygments_lexer: + string += " {}".format(pygments_lexer) + string += "\n" + metadata = from_nbnode(cell.metadata) + if metadata: + string += "---\n" + string += yaml.safe_dump(metadata) + string += "---\n" + string += cell.source + if not cell.source.endswith("\n"): + string += "\n" + string += "```\n" + last_cell_md = False + + else: + raise NotImplementedError("cell {}, type: {}".format(i, cell.cell_type)) + + return string.rstrip() + "\n" diff --git a/tests/test_convert/test_basic.ipynb b/tests/roundtrip/basic.ipynb similarity index 88% rename from tests/test_convert/test_basic.ipynb rename to tests/roundtrip/basic.ipynb index ddf955a6..c8888b87 100644 --- a/tests/test_convert/test_basic.ipynb +++ b/tests/roundtrip/basic.ipynb @@ -47,6 +47,10 @@ } ], "metadata": { + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + }, "orphan": true }, "nbformat": 4, diff --git a/tests/roundtrip/basic.mystnb b/tests/roundtrip/basic.mystnb new file mode 100644 index 00000000..347fe733 --- /dev/null +++ b/tests/roundtrip/basic.mystnb @@ -0,0 +1,27 @@ +--- +language_info: + name: python + pygments_lexer: ipython3 +nbformat: 4 +nbformat_minor: 4 +orphan: true +--- + +a + +b +c +```{nb-code} ipython3 +--- +tags: +- hide-code +--- +a = 1 +print(a) +``` + +c + ++++ {"tags": ["hide-cell"]} + +d diff --git a/tests/test_convert.py b/tests/test_convert.py index 9f98572f..18513caf 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -1,34 +1,20 @@ -from textwrap import dedent -import nbformat -from myst_nb.convert import myst_to_nb - - -def test_basic(file_regression): - text = dedent( - """\ - --- - orphan: true - --- +from pathlib import Path - a +import nbformat +from myst_nb.convert import myst_to_nb, nb_to_myst - b - c - ```{nb-cell} - --- - tags: ["hide-code"] - --- - a = 1 - print(a) - ``` +SOURCEDIR = Path(__file__).parent.joinpath("roundtrip") - c - +++ {"tags": ["hide-cell"]} +def test_myst_to_nb(file_regression): + text = SOURCEDIR.joinpath("basic.mystnb").read_text() + notebook = myst_to_nb(text, directive="nb-code") + file_regression.check( + nbformat.writes(notebook), fullpath=SOURCEDIR.joinpath("basic.ipynb") + ) - d - """ - ) - notebook = myst_to_nb(text, directive="nb-cell") - file_regression.check(nbformat.writes(notebook), extension=".ipynb") +def test_nb_to_myst(file_regression): + text = SOURCEDIR.joinpath("basic.ipynb").read_text() + output = nb_to_myst(nbformat.reads(text, 4), directive="nb-code") + file_regression.check(output, fullpath=SOURCEDIR.joinpath("basic.mystnb"))