Add documentation of myst-nb format

demo/Benchmarking Jupytext.py

@@ -32,8 +32,8 @@
 
 # Let's see if we have myst-parser installed here
 try:
-    jupytext.writes(notebook, fmt='mystnb')
-    JUPYTEXT_FORMATS.append('mystnb')
+    jupytext.writes(notebook, fmt='myst')
+    JUPYTEXT_FORMATS.append('myst')
 except jupytext.formats.JupytextFormatError as err:
     print(str(err))
 

tests/notebooks/mirror/ipynb_to_myst/World population.mystnb → demo/World population.myst.md

@@ -1,10 +1,15 @@
 ---
+jupytext:
+  formats: ipynb,.pct.py:percent,.lgt.py:light,.spx.py:sphinx,md,Rmd,.pandoc.md:pandoc,.myst.md:myst
+  text_representation:
+    extension: '.md'
+    format_name: myst
+    format_version: '0.7'
+    jupytext_version: 1.4.0+dev
 kernelspec:
   display_name: Python 3
   language: python
   name: python3
-nbformat: 4
-nbformat_minor: 2
 ---
 
 # A quick insight at world population
@@ -15,7 +20,7 @@ In the below we retrieve population data from the
 [World Bank](http://www.worldbank.org/)
 using the [wbdata](https://github.com/OliverSherouse/wbdata) python package
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 import pandas as pd
 import wbdata as wb
 
@@ -26,15 +31,15 @@ pd.options.display.max_columns = 20
 Corresponding indicator is found using search method - or, directly,
 the World Bank site.
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 wb.search_indicators('Population, total')  # SP.POP.TOTL
 # wb.search_indicators('area')
 # => https://data.worldbank.org/indicator is easier to use
 ```
 
 Now we download the population data
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 indicators = {'SP.POP.TOTL': 'Population, total',
               'AG.SRF.TOTL.K2': 'Surface area (sq. km)',
               'AG.LND.TOTL.K2': 'Land area (sq. km)',
@@ -45,20 +50,20 @@ data
 
 World is one of the countries
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 data.loc['World']
 ```
 
 Can we classify over continents?
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 data.loc[(slice(None), '2017-01-01'), :]['Population, total'].dropna(
 ).sort_values().tail(60).index.get_level_values('country')
 ```
 
 Extract zones manually (in order of increasing population)
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 zones = ['North America', 'Middle East & North Africa',
          'Latin America & Caribbean', 'Europe & Central Asia',
          'Sub-Saharan Africa', 'South Asia',
@@ -67,19 +72,19 @@ zones = ['North America', 'Middle East & North Africa',
 
 And extract population information (and check total is right)
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 population = data.loc[zones]['Population, total'].swaplevel().unstack()
 population = population[zones]
 assert all(data.loc['World']['Population, total'] == population.sum(axis=1))
 ```
 
 ## Stacked area plot with matplotlib
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 import matplotlib.pyplot as plt
 ```
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 plt.clf()
 plt.figure(figsize=(10, 5), dpi=100)
 plt.stackplot(population.index, population.values.T / 1e9)
@@ -97,14 +102,14 @@ selected legends) are
 [on their way](https://github.com/plotly/plotly.js/pull/2960) at Plotly. For
 now we just do a stacked bar plot.
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 import plotly.offline as offline
 import plotly.graph_objs as go
 
 offline.init_notebook_mode()
 ```
 
-```{nb-code} ipython3
+```{code-cell} ipython3
 bars = [go.Bar(x=population.index, y=population[zone], name=zone)
         for zone in zones]
 fig = go.Figure(data=bars,

docs/formats.md

@@ -83,6 +83,77 @@ See for instance how our `World population.ipynb` notebook is [represented](http
 
 If you wish to use that format, please install `pandoc` in version 2.7.2 or above, with e.g. `conda install pandoc -c conda-forge`.
 
+### MyST Markdown
+
+[MyST (Markedly Structured Text)][myst-parser] is a markdown flavor that "implements the best parts of reStructuredText". It provides a way to call Sphinx directives and roles from within Markdown,
+using a *slight* extension of CommonMark markdown.
+[MyST-NB][myst-nb] builds on this markdown flavor, to offer direct conversion of Jupyter Notebooks into Sphinx documents.
+
+Similar to the jupytext Markdown format, MyST Markdown uses code blocks to contain code cells.
+The difference though, is that the metadata is contained in a YAML block:
+
+````md
+```{code-cell} ipython3
+---
+other:
+  more: true
+tags: [hide-output, show-input]
+---
+
+print("Hallo!")
+```
+````
+
+The `ipython3` here is purely optional, as an aide for syntax highlighting.
+In the round-trip, it is copied from `notebook.metadata.language_info.pygments_lexer`.
+
+Also, where possible the conversion will use the short-hand metadata format
+(see the [MyST guide](https://myst-parser.readthedocs.io/en/latest/using/syntax.html#parameterizing-directives)):
+
+````md
+```{code-cell} ipython3
+:tags: [hide-output, show-input]
+
+print("Hallo!")
+```
+````
+
+Raw cells are also represented in a similare fashion:
+
+````md
+```{raw-cell}
+:raw_mimetype: text/html
+
+<b>Bold text<b>
+```
+````
+
+Markdown cells are not wrapped. If a markdown cell has metadata, or
+directly proceeds another markdown cell, then a [block break] will be inserted
+above it, with an (optional) single line JSON representation of the metadata:
+
+```md
++++ {"slide": true}
+
+This is a markdown cell with metadata
+
++++
+
+This is a new markdown cell with no metadata
+```
+
+See for instance how our `World population.ipynb` notebook is [represented](https://github.com/mwouts/jupytext/blob/master/demo/World%20population.myst.md#) in the `myst` format.
+
+If you wish to use that format, please install `conda install -c conda-forge myst-parser`,
+or `pip install jupytext[myst]`.
+
+**Tip**: You can use the [myst-highlight] VS Code extension to provide better syntax highlighting for this format.
+
+[myst-parser]: https://myst-parser.readthedocs.io
+[myst-nb]: https://myst-nb.readthedocs.io
+[block break]: https://myst-parser.readthedocs.io/en/latest/using/syntax.html#block-breaks
+[myst-highlight]: https://marketplace.visualstudio.com/items?itemName=ExecutableBookProject.myst-highlight
+
 ## Notebooks as scripts
 
 ### The `light` format

jupytext/formats.py

@@ -24,6 +24,7 @@
     is_myst_available,
     myst_version,
     myst_extensions,
+    matches_mystnb,
 )
 
 
@@ -196,6 +197,8 @@ def get_format_implementation(ext, format_name=None):
     if formats_for_extension:
         if ext in ['.md', '.markdown'] and format_name == 'pandoc':
             raise JupytextFormatError('Please install pandoc>=2.7.2')
+        if ext in myst_extensions() and format_name == MYST_FORMAT_NAME:
+            raise JupytextFormatError('Please install myst-parser')
 
         raise JupytextFormatError("Format '{}' is not associated to extension '{}'. "
                                   "Please choose one of: {}.".format(format_name, ext,
@@ -229,6 +232,8 @@ def read_format_from_metadata(text, ext):
 
 def guess_format(text, ext):
     """Guess the format and format options of the file, given its extension and content"""
+    if matches_mystnb(text, ext):
+        return MYST_FORMAT_NAME, {}
     lines = text.splitlines()
 
     metadata = read_metadata(text, ext)
@@ -463,11 +468,14 @@ def long_form_one_format(jupytext_format, metadata=None, update=None, auto_ext_r
     if not jupytext_format:
         return {}
 
-    common_name_to_ext = {'notebook': 'ipynb',
-                          'rmarkdown': 'Rmd',
-                          'markdown': 'md',
-                          'script': 'auto',
-                          'c++': 'cpp'}
+    common_name_to_ext = {
+        'notebook': 'ipynb',
+        'rmarkdown': 'Rmd',
+        'markdown': 'md',
+        'script': 'auto',
+        'c++': 'cpp',
+        'myst': 'md'
+    }
     if jupytext_format.lower() in common_name_to_ext:
         jupytext_format = common_name_to_ext[jupytext_format.lower()]
 
@@ -539,7 +547,7 @@ def short_form_one_format(jupytext_format):
 
     if jupytext_format.get('format_name'):
         if jupytext_format['extension'] not in ['.md', '.markdown', '.Rmd'] or \
-                jupytext_format['format_name'] == 'pandoc':
+                jupytext_format['format_name'] in ['pandoc', MYST_FORMAT_NAME]:
             fmt = fmt + ':' + jupytext_format['format_name']
 
     return fmt

jupytext/jupytext.py

@@ -19,7 +19,7 @@
 from .languages import default_language_from_metadata_and_ext, set_main_and_cell_language
 from .pep8 import pep8_lines_between_cells
 from .pandoc import md_to_notebook, notebook_to_md
-from .myst import myst_extensions, myst_to_notebook, notebook_to_myst
+from .myst import myst_extensions, myst_to_notebook, notebook_to_myst, MYST_FORMAT_NAME
 
 
 class TextNotebookConverter(NotebookReader, NotebookWriter):
@@ -55,7 +55,7 @@ def reads(self, s, **_):
         if self.fmt.get('format_name') == 'pandoc':
             return md_to_notebook(s)
 
-        if self.ext in myst_extensions():
+        if self.fmt.get('format_name') == MYST_FORMAT_NAME:
             return myst_to_notebook(s)
 
         lines = s.splitlines()
@@ -127,7 +127,7 @@ def writes(self, nb, metadata=None, **kwargs):
                 metadata=metadata,
                 cells=cells))
 
-        if self.ext in myst_extensions():
+        if self.fmt.get('format_name') == MYST_FORMAT_NAME or self.ext in myst_extensions(no_md=True):
             pygments_lexer = metadata.get("language_info", {}).get("pygments_lexer", None)
             metadata = insert_jupytext_info_and_filter_metadata(metadata, self.ext, self.implementation)