make myst's default extension .md

demo/Benchmarking Jupytext.py

@@ -32,8 +32,8 @@
 
 # Let's see if we have myst-parser installed here
 try:
-    jupytext.writes(notebook, fmt='mystnb')
-    JUPYTEXT_FORMATS.append('mystnb')
+    jupytext.writes(notebook, fmt='myst')
+    JUPYTEXT_FORMATS.append('myst')
 except jupytext.formats.JupytextFormatError as err:
     print(str(err))
 

demo/World population.mnb → demo/World population.myst.md

@@ -1,9 +1,9 @@
 ---
 jupytext:
-  formats: ipynb,.pct.py:percent,.lgt.py:light,.spx.py:sphinx,md,Rmd,.pandoc.md:pandoc
+  formats: ipynb,.pct.py:percent,.lgt.py:light,.spx.py:sphinx,md,Rmd,.pandoc.md:pandoc,.myst.md:myst
   text_representation:
-    extension: .mnb
-    format_name: mystnb
+    extension: '.md'
+    format_name: myst
     format_version: 0.7.1
     jupytext_version: 1.4.0+dev
 kernelspec:

docs/formats.md

@@ -83,13 +83,13 @@ See for instance how our `World population.ipynb` notebook is [represented](http
 
 If you wish to use that format, please install `pandoc` in version 2.7.2 or above, with e.g. `conda install pandoc -c conda-forge`.
 
-### MyST-NB Markdown
+### MyST Markdown
 
 [MyST (Markedly Structured Text)][myst-parser] is a markdown flavor that "implements the best parts of reStructuredText". It provides a way to call Sphinx directives and roles from within Markdown,
 using a *slight* extension of CommonMark markdown.
 [MyST-NB][myst-nb] builds on this markdown flavor, to offer direct conversion of Jupyter Notebooks into Sphinx documents.
 
-Similar to the Markdown format, MyST-NB uses code blocks to contain code cells.
+Similar to the jupytext Markdown format, MyST Markdown uses code blocks to contain code cells.
 The difference though, is that the metadata is contained in a YAML block:
 
 ````md
@@ -142,7 +142,7 @@ This is a markdown cell with metadata
 This is a new markdown cell with no metadata
 ```
 
-See for instance how our `World population.ipynb` notebook is [represented](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.mnb#) in the `mystnb` format.
+See for instance how our `World population.ipynb` notebook is [represented](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.myst.md#) in the `myst` format.
 
 If you wish to use that format, please install `conda install -c conda-forge myst-parser`,
 or `pip install jupytext[myst]`.

jupytext/formats.py

@@ -24,6 +24,7 @@
     is_myst_available,
     myst_version,
     myst_extensions,
+    matches_mystnb,
 )
 
 
@@ -231,6 +232,8 @@ def read_format_from_metadata(text, ext):
 
 def guess_format(text, ext):
     """Guess the format and format options of the file, given its extension and content"""
+    if matches_mystnb(text, ext):
+        return MYST_FORMAT_NAME, {}
     lines = text.splitlines()
 
     metadata = read_metadata(text, ext)
@@ -465,11 +468,14 @@ def long_form_one_format(jupytext_format, metadata=None, update=None, auto_ext_r
     if not jupytext_format:
         return {}
 
-    common_name_to_ext = {'notebook': 'ipynb',
-                          'rmarkdown': 'Rmd',
-                          'markdown': 'md',
-                          'script': 'auto',
-                          'c++': 'cpp'}
+    common_name_to_ext = {
+        'notebook': 'ipynb',
+        'rmarkdown': 'Rmd',
+        'markdown': 'md',
+        'script': 'auto',
+        'c++': 'cpp',
+        'myst': 'md'
+    }
     if jupytext_format.lower() in common_name_to_ext:
         jupytext_format = common_name_to_ext[jupytext_format.lower()]
 

jupytext/jupytext.py

@@ -19,7 +19,7 @@
 from .languages import default_language_from_metadata_and_ext, set_main_and_cell_language
 from .pep8 import pep8_lines_between_cells
 from .pandoc import md_to_notebook, notebook_to_md
-from .myst import myst_extensions, myst_to_notebook, notebook_to_myst
+from .myst import myst_extensions, myst_to_notebook, notebook_to_myst, MYST_FORMAT_NAME
 
 
 class TextNotebookConverter(NotebookReader, NotebookWriter):
@@ -55,7 +55,7 @@ def reads(self, s, **_):
         if self.fmt.get('format_name') == 'pandoc':
             return md_to_notebook(s)
 
-        if self.ext in myst_extensions():
+        if self.fmt.get('format_name') == MYST_FORMAT_NAME:
             return myst_to_notebook(s)
 
         lines = s.splitlines()
@@ -127,7 +127,7 @@ def writes(self, nb, metadata=None, **kwargs):
                 metadata=metadata,
                 cells=cells))
 
-        if self.ext in myst_extensions():
+        if self.fmt.get('format_name') == MYST_FORMAT_NAME or self.ext in myst_extensions(no_md=True):
             pygments_lexer = metadata.get("language_info", {}).get("pygments_lexer", None)
             metadata = insert_jupytext_info_and_filter_metadata(metadata, self.ext, self.implementation)
 

jupytext/myst.py

@@ -7,12 +7,13 @@
 import nbformat as nbf
 import yaml
 
-MYST_FORMAT_NAME = "mystnb"
+MYST_FORMAT_NAME = "myst"
 CODE_DIRECTIVE = "code-cell"
 RAW_DIRECTIVE = "raw-cell"
 
 
 def is_myst_available():
+    """Whether the myst-parser package is available."""
     try:
         import myst_parser  # noqa
     except ImportError:
@@ -21,13 +22,50 @@ def is_myst_available():
 
 
 def myst_version():
+    """The version of myst parser."""
     from myst_parser import __version__
 
     return __version__
 
 
-def myst_extensions():
-    return [".myst.md", ".mnb"]
+def myst_extensions(no_md=False):
+    """The allowed extensions for the myst format."""
+    if no_md:
+        return [".myst", ".mystnb", ".mnb"]
+    return [".md", ".myst", ".mystnb", ".mnb"]
+
+
+def matches_mystnb(text, ext=None, requires_meta=True, require_non_md=True):
+    """Attempt to distinguish a file as mystnb, only given its extension and content.
+
+    :param ext: the extension of the file
+    :param requires_meta: requires the file to contain top matter metadata
+    :param require_non_md: whether to require that a non-markdown cell is present
+    """
+    if ext and (ext+".").rsplit(".", 1)[1] in ["myst", "mystnb"]:
+        return True
+    if requires_meta and not text.startswith("---"):
+        return False
+    try:
+        nb = myst_to_notebook(text, ignore_bad_meta=True)
+    except Exception:
+        return False
+
+    from jupytext.formats import format_name_for_ext
+
+    # Is the format information available in the jupytext text representation?
+    try:
+        format_name = format_name_for_ext(nb.metadata, ext or ".md")
+    except AttributeError:
+        pass
+    else:
+        if format_name == MYST_FORMAT_NAME:
+            return True
+
+    if require_non_md and not any(c.cell_type != "markdown" for c in nb.cells):
+        return False
+
+    return True
 
 
 class CompactDumper(yaml.SafeDumper):
@@ -96,13 +134,14 @@ def _fmt_md(text):
 
 
 def myst_to_notebook(
-    text, code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE,
+    text, code_directive=CODE_DIRECTIVE, raw_directive=RAW_DIRECTIVE, ignore_bad_meta=False
 ):
     """Convert text written in the myst format to a notebook.
 
     :param text: the file text
     :param code_directive: the name of the directive to search for containing code cells
     :param raw_directive: the name of the directive to search for containing raw cells
+    :param ignore_bad_meta: ignore metadata that cannot be parsed as JSON/YAML
 
     NOTE: we assume here that all of these directives are at the top-level,
     i.e. not nested in other directives.
@@ -136,11 +175,12 @@ def myst_to_notebook(
     try:
         set_parse_context(parse_context)
         doc = Document.read(lines, front_matter=True)
+        metadata_nb = {}
         try:
             metadata_nb = doc.front_matter.get_data() if doc.front_matter else {}
         except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error:
-            raise MystMetadataParsingError("Notebook metadata: {}".format(error))
-
+            if not ignore_bad_meta:
+                raise MystMetadataParsingError("Notebook metadata: {}".format(error))
         nbf_version = nbf.v4
         kwargs = {"metadata": nbf.from_dict(metadata_nb)}
         notebook = nbf_version.new_notebook(**kwargs)
@@ -161,20 +201,23 @@ def myst_to_notebook(
                         )
                     )
                 if token.content:
+                    md_metadata = {}
                     try:
                         md_metadata = json.loads(token.content.strip())
                     except Exception as err:
-                        raise MystMetadataParsingError(
-                            "markdown cell {0} at {1} could not be read: {2}".format(
-                                len(notebook.cells) + 1, token.position, err
+                        if not ignore_bad_meta:
+                            raise MystMetadataParsingError(
+                                "markdown cell {0} at {1} could not be read: {2}".format(
+                                    len(notebook.cells) + 1, token.position, err
+                                )
                             )
-                        )
                     if not isinstance(md_metadata, dict):
-                        raise MystMetadataParsingError(
-                            "markdown cell {0} at {1} is not a dict".format(
-                                len(notebook.cells) + 1, token.position
+                        if not ignore_bad_meta:
+                            raise MystMetadataParsingError(
+                                "markdown cell {0} at {1} is not a dict".format(
+                                    len(notebook.cells) + 1, token.position
+                                )
                             )
-                        )
                 else:
                     md_metadata = {}
                 current_line = token.position.line_start
@@ -187,6 +230,7 @@ def myst_to_notebook(
                 # this is reserved for the optional lexer name
                 # TODO: could log warning about if token.arguments != lexer name
 
+                options, body_lines = {}, []
                 try:
                     _, options, body_lines = parse_directive_text(
                         directive_class=MockDirective,
@@ -195,11 +239,12 @@ def myst_to_notebook(
                         validate_options=False,
                     )
                 except DirectiveParsingError as err:
-                    raise MystMetadataParsingError(
-                        "Code cell {0} at {1} could not be read: {2}".format(
-                            len(notebook.cells) + 1, token.position, err
+                    if not ignore_bad_meta:
+                        raise MystMetadataParsingError(
+                            "Code cell {0} at {1} could not be read: {2}".format(
+                                len(notebook.cells) + 1, token.position, err
+                            )
                         )
-                    )
 
                 md_source = _fmt_md(
                     "".join(lines.lines[current_line:token.position.line_start - 1])