Encapsulate code cells with triple backticks with +1 backtick in Markdown

docs/CHANGELOG.md

@@ -1,14 +1,15 @@
 Jupytext ChangeLog
 ==================
 
-1.9.2-dev (???)
----------------
+1.10.0-dev (???)
+----------------
 
 **Changed**
 - Jupytext does not work properly with the new cell ids of the version 4.5 of `nbformat>=5.1.0` yet, so we added the requirement `nbformat<=5.0.8` ([#715](https://github.com/mwouts/jupytext/issues/715))
 - `jupytext --sync` only updates the timestamp of the text file (not the file itself) when that file is the most recent ([#698](https://github.com/mwouts/jupytext/issues/698))
 
 **Fixed**
+- Code cells that contain triple backticks (or more) are now encapsulated with four backticks (or more) in the Markdown and MyST Markdown formats. The version number for the Markdown format was increased to 1.3, and the version number for the MyST Markdown format was increased to 0.13 ([#712](https://github.com/mwouts/jupytext/issues/712))
 - Indented magic commands are supported ([#694](https://github.com/mwouts/jupytext/issues/694))
 - Jupytext will issue an informative error or warning on notebooks in a version of nbformat that is not known to be supported ([#681](https://github.com/mwouts/jupytext/issues/681), [#715](https://github.com/mwouts/jupytext/issues/715))
 

jupytext/cell_reader.py

@@ -309,7 +309,7 @@ class MarkdownCellReader(BaseCellReader):
 
     comment = ""
     start_code_re = re.compile(
-        r"^```(\s*)({})($|\s.*$)".format(
+        r"^```(`*)(\s*)({})($|\s.*$)".format(
             "|".join(_JUPYTER_LANGUAGES_LOWER_AND_UPPER).replace("+", "\\+")
         )
     )
@@ -362,7 +362,10 @@ def metadata_and_language_from_option_line(self, line):
 
     def options_to_metadata(self, options):
         if isinstance(options, tuple):
-            options = " ".join(options)
+            self.end_code_re = re.compile("```" + options[0])
+            options = " ".join(options[1:])
+        else:
+            self.end_code_re = re.compile(r"^```\s*$")
         self.cell_metadata_json = self.cell_metadata_json or is_json_metadata(options)
         return text_to_metadata(options)
 
@@ -441,6 +444,9 @@ def find_cell_end(self, lines):
                     prev_blank = 0
         else:
             self.cell_type = "code"
+            # At some point we could remove the below, in which we make sure not to break language strings
+            # into multiple cells (#419). Indeed, now that the markdown cell uses one extra backtick (#712)
+            # we should not have the issue any more
             parser = StringParser(self.language or self.default_language)
             for i, line in enumerate(lines):
                 # skip cell header

jupytext/cell_to_text.py

@@ -29,6 +29,22 @@ def cell_source(cell):
     return source.splitlines()
 
 
+def three_backticks_or_more(lines):
+    """Return a string with enough backticks to encapsulate the given code cell in Markdown
+    cf. https://github.com/mwouts/jupytext/issues/712"""
+    code_cell_delimiter = "```"
+    for line in lines:
+        if not line.startswith(code_cell_delimiter):
+            continue
+        for char in line[len(code_cell_delimiter) :]:
+            if char != "`":
+                break
+            code_cell_delimiter += "`"
+        code_cell_delimiter += "`"
+
+    return code_cell_delimiter
+
+
 class BaseCellExporter(object):
     """A class that represent a notebook cell as text"""
 
@@ -222,7 +238,8 @@ def code_to_text(self):
             return self.html_comment(self.metadata, "raw")
 
         options = metadata_to_text(self.language, self.metadata)
-        return ["```" + options] + source + ["```"]
+        code_cell_delimiter = three_backticks_or_more(self.source)
+        return [code_cell_delimiter + options] + source + [code_cell_delimiter]
 
 
 class RMarkdownCellExporter(MarkdownCellExporter):

jupytext/formats.py

@@ -83,7 +83,8 @@ def __init__(
             # Version 1.1 on 2019-03-24 - jupytext v1.1.0 : Markdown regions and cell metadata
             # Version 1.2 on 2019-09-21 - jupytext v1.3.0 : Raw regions are now encoded with HTML comments (#321)
             # and by default, cell metadata use the key=value representation (#347)
-            current_version_number="1.2",
+            # Version 1.3 on 2021-01-24 - jupytext v1.10.0 : Code cells may start with more than three backticks (#712)
+            current_version_number="1.3",
             min_readable_version_number="1.0",
         ),
         NotebookFormatDescription(

jupytext/myst.py

@@ -10,6 +10,8 @@
 import nbformat as nbf
 import yaml
 
+from .cell_to_text import three_backticks_or_more
+
 try:
     from markdown_it import MarkdownIt
     from mdit_py_plugins.front_matter import front_matter_plugin
@@ -38,7 +40,7 @@ def raise_if_myst_is_not_available():
 
 def myst_version():
     """The version of myst."""
-    return 0.12
+    return 0.13
 
 
 def myst_extensions(no_md=False):
@@ -396,8 +398,10 @@ def notebook_to_myst(
             last_cell_md = True
 
         elif cell.cell_type in ["code", "raw"]:
-            string += "\n```{}".format(
-                code_directive if cell.cell_type == "code" else raw_directive
+            cell_delimiter = three_backticks_or_more(cell.source.splitlines())
+            string += "\n{}{}".format(
+                cell_delimiter,
+                code_directive if cell.cell_type == "code" else raw_directive,
             )
             if pygments_lexer and cell.cell_type == "code":
                 string += " {}".format(pygments_lexer)
@@ -410,7 +414,7 @@ def notebook_to_myst(
             string += cell.source
             if not cell.source.endswith("\n"):
                 string += "\n"
-            string += "```\n"
+            string += cell_delimiter + "\n"
             last_cell_md = False
 
         else:

jupytext/version.py

@@ -1,3 +1,3 @@
 """Jupytext's version number"""
 
-__version__ = "1.9.1+dev"
+__version__ = "1.10.0-dev"

tests/test_markdown_in_code_cells.py

@@ -0,0 +1,129 @@
+"""Issue #712"""
+from nbformat.v4.nbbase import new_code_cell, new_notebook
+
+from jupytext import reads, writes
+from jupytext.cell_to_text import three_backticks_or_more
+from jupytext.compare import compare, compare_notebooks
+
+from .utils import requires_myst
+
+
+def test_three_backticks_or_more():
+    assert three_backticks_or_more([""]) == "```"
+    assert three_backticks_or_more(["``"]) == "```"
+    assert three_backticks_or_more(["```python"]) == "````"
+    assert three_backticks_or_more(["```"]) == "````"
+    assert three_backticks_or_more(["`````python"]) == "``````"
+    assert three_backticks_or_more(["`````"]) == "``````"
+
+
+def test_triple_backticks_in_code_cell(
+    no_jupytext_version_number,
+    nb=new_notebook(
+        metadata={"main_language": "python"},
+        cells=[
+            new_code_cell(
+                '''a = """
+```
+foo
+```
+"""'''
+            )
+        ],
+    ),
+    text='''---
+jupyter:
+  jupytext:
+    main_language: python
+---
+
+````python
+a = """
+```
+foo
+```
+"""
+````
+''',
+):
+    actual_text = writes(nb, fmt="md")
+    compare(actual_text, text)
+
+    actual_nb = reads(text, fmt="md")
+    compare_notebooks(actual_nb, nb)
+
+
+@requires_myst
+def test_triple_backticks_in_code_cell_myst(
+    no_jupytext_version_number,
+    nb=new_notebook(
+        metadata={"main_language": "python"},
+        cells=[
+            new_code_cell(
+                '''a = """
+```
+foo
+```
+"""'''
+            )
+        ],
+    ),
+    text='''---
+jupytext:
+  main_language: python
+---
+
+````{code-cell}
+a = """
+```
+foo
+```
+"""
+````
+''',
+):
+    actual_text = writes(nb, fmt="md:myst")
+    compare(actual_text, text)
+
+    actual_nb = reads(text, fmt="md:myst")
+    compare_notebooks(actual_nb, nb)
+
+
+def test_alternate_tree_four_five_backticks(
+    no_jupytext_version_number,
+    nb=new_notebook(
+        metadata={"main_language": "python"},
+        cells=[
+            new_code_cell('a = """\n```\n"""'),
+            new_code_cell("b = 2"),
+            new_code_cell('c = """\n````\n"""'),
+        ],
+    ),
+    text='''---
+jupyter:
+  jupytext:
+    main_language: python
+---
+
+````python
+a = """
+```
+"""
+````
+
+```python
+b = 2
+```
+
+`````python
+c = """
+````
+"""
+`````
+''',
+):
+    actual_text = writes(nb, fmt="md")
+    compare(actual_text, text)
+
+    actual_nb = reads(text, fmt="md")
+    compare_notebooks(actual_nb, nb)

tests/test_read_simple_markdown.py

@@ -779,8 +779,9 @@ def test_markdown_cell_with_code_inside_multiline_string_419(
 ```
 ''',
 ):
+    """A code cell containing triple backticks is converted to a code cell encapsulated with four backticks"""
     nb = jupytext.reads(text, "md")
-    compare(jupytext.writes(nb, "md"), text)
+    compare(jupytext.writes(nb, "md"), "`" + text[:-1] + "`\n")
     assert len(nb.cells) == 1