Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

👌 Fix myst-parser install issues #599

Closed
wants to merge 8 commits into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 1 addition & 9 deletions .github/workflows/continuous-integration-conda.yml
Original file line number Diff line number Diff line change
@@ -6,11 +6,7 @@ jobs:
strategy:
matrix:
os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
python-version: [2.7, 3.7]
exclude:
# This one fails, cf. https://github.com/mwouts/jupytext/runs/736344037
- os: windows-latest
python-version: 2.7
python-version: [3.7]
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
@@ -56,17 +52,13 @@ jobs:
shell: pwsh
run: |
$ErrorActionPreference='silentlycontinue'
# Install mock on Python 2.7
conda install mock --freeze-installed
# install black if available (Python 3.6 and above), and autopep8 for testing the pipe mode
conda install black --freeze-installed
# install isort from source
pip install git+https://github.com/timothycrosley/isort.git
conda install autopep8 --freeze-installed
# install sphinx_gallery and matplotlib if available
conda install sphinx-gallery --freeze-installed
# myst-parser
conda install 'myst-parser>=0.8' 'myst-parser<0.9' --freeze-installed
exit 0
- name: Conda list
shell: pwsh
13 changes: 11 additions & 2 deletions .github/workflows/continuous-integration-pip.yml
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
name: CI (pip)
on: [push]
on:
push:
branches: [master]
tags:
- '*'
pull_request:

jobs:
build:
strategy:
matrix:
python-version: [2.7, 3.5, 3.6, 3.7, 3.8]
python-version: [3.6, 3.7, 3.8]
runs-on: ubuntu-latest
steps:
- name: Checkout
@@ -14,6 +19,10 @@ jobs:
uses: actions/setup-python@v1
with:
python-version: ${{ matrix.python-version }}
- name: install pandoc
uses: r-lib/actions/setup-pandoc@v1
with:
pandoc-version: '2.9'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -14,6 +14,8 @@ future
gits*
.ipynb_checkpoints
docs/_build
.DS_Store
.tox

# Will be created by postBuild
demo/get_started.ipynb
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -23,6 +23,6 @@ repos:
- id: flake8

- repo: https://github.com/psf/black
rev: stable
rev: 19.10b0
hooks:
- id: black
3 changes: 1 addition & 2 deletions jupytext/formats.py
Original file line number Diff line number Diff line change
@@ -35,7 +35,6 @@
from .magics import is_magic
from .myst import (
MYST_FORMAT_NAME,
is_myst_available,
myst_version,
myst_extensions,
matches_mystnb,
@@ -291,7 +290,7 @@ def guess_format(text, ext):
if "text_representation" in metadata.get("jupytext", {}):
return format_name_for_ext(metadata, ext), {}

if is_myst_available() and matches_mystnb(text, ext):
if matches_mystnb(text, ext):
return MYST_FORMAT_NAME, {}

lines = text.splitlines()
140 changes: 75 additions & 65 deletions jupytext/myst.py
Original file line number Diff line number Diff line change
@@ -4,49 +4,25 @@
"""
import json
import warnings
import re
from textwrap import dedent

import nbformat as nbf
import yaml
from .reraise import reraise

try:
import myst_parser
from myst_parser.main import default_parser
from myst_parser.parse_directives import DirectiveParsingError, parse_directive_text
except ImportError as err:
myst_parser = None
DirectiveParsingError = Exception
default_parser = parse_directive_text = reraise(err)
from markdown_it import MarkdownIt
from markdown_it.extensions.front_matter import front_matter_plugin
from markdown_it.extensions.myst_blocks import myst_block_plugin
from markdown_it.extensions.myst_role import myst_role_plugin

MYST_FORMAT_NAME = "myst"
CODE_DIRECTIVE = "{code-cell}"
RAW_DIRECTIVE = "{raw-cell}"


def is_myst_available():
"""Whether the myst-parser package is available."""
if myst_parser is None:
return False
major, minor = myst_parser.__version__.split(".")[:2]
if int(major) < 1 and int(minor) < 8:
warnings.warn("The installed myst-parser version is less than the required 0.8")
return False
return True


def raise_if_myst_is_not_available():
if not is_myst_available():
raise ImportError(
"The MyST Markdown format requires 'myst_parser>=0.8'. "
"Install it with e.g. 'pip install jupytext[myst]'"
)


def myst_version():
"""The major version of myst parser."""
if is_myst_available():
return ".".join(myst_parser.__version__.split(".")[:2])
return "N/A"
"""The version of myst."""
return 0.12


def myst_extensions(no_md=False):
@@ -56,6 +32,20 @@ def myst_extensions(no_md=False):
return [".md", ".myst", ".mystnb", ".mnb"]


def get_parser():
"""Return the markdown-it parser to use."""
parser = (
MarkdownIt("commonmark")
.enable("table")
.use(front_matter_plugin)
.use(myst_block_plugin)
.use(myst_role_plugin)
# we only need to parse block level components (for efficiency)
.disable("inline", True)
)
return parser


def matches_mystnb(
text,
ext=None,
@@ -79,9 +69,7 @@ def matches_mystnb(
return False

try:
# parse markdown file up to the block level (i.e. don't worry about inline text)
parser = default_parser("html", disable_syntax=["inline"])
tokens = parser.parse(text + "\n")
tokens = get_parser().parse(text + "\n")
except (TypeError, ValueError) as err:
warnings.warn("myst-parse failed unexpectedly: {}".format(err))
return False
@@ -164,17 +152,61 @@ def from_nbnode(value):
return value


class MockDirective:
option_spec = {"options": True}
required_arguments = 0
optional_arguments = 1
has_content = True


class MystMetadataParsingError(Exception):
"""Error when parsing metadata from myst formatted text"""


def read_fenced_cell(token, cell_index, cell_type):
"""Parse (and validate) the full directive text."""
content = token.content
error_msg = "{0} cell {1} at line {2} could not be read: ".format(
cell_type, cell_index, token.map[0] + 1
)

body_lines, options = parse_directive_options(content, error_msg)

# remove first line of body if blank
# this is to allow space between the options and the content
if body_lines and not body_lines[0].strip():
body_lines = body_lines[1:]

return options, body_lines


def parse_directive_options(content: str, error_msg: str):
"""Parse (and validate) the directive option section."""
options = {}
if content.startswith("---"):
content = "\n".join(content.splitlines()[1:])
match = re.search(r"^-{3,}", content, re.MULTILINE)
if match:
yaml_block = content[: match.start()]
content = content[match.end() + 1 :]
else:
yaml_block = content
content = ""
yaml_block = dedent(yaml_block)
try:
options = yaml.safe_load(yaml_block) or {}
except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error:
raise MystMetadataParsingError(error_msg + "Invalid YAML; " + str(error))
elif content.lstrip().startswith(":"):
content_lines = content.splitlines() # type: list
yaml_lines = []
while content_lines:
if not content_lines[0].lstrip().startswith(":"):
break
yaml_lines.append(content_lines.pop(0).lstrip()[1:])
yaml_block = "\n".join(yaml_lines)
content = "\n".join(content_lines)
try:
options = yaml.safe_load(yaml_block) or {}
except (yaml.parser.ParserError, yaml.scanner.ScannerError) as error:
raise MystMetadataParsingError(error_msg + "Invalid YAML; " + str(error))

return content.splitlines(), options


def strip_blank_lines(text):
"""Remove initial blank lines"""
text = text.rstrip()
@@ -183,24 +215,6 @@ def strip_blank_lines(text):
return text


def read_fenced_cell(token, cell_index, cell_type):
"""Return cell options and body"""
try:
_, options, body_lines = parse_directive_text(
directive_class=MockDirective,
argument_str="",
content=token.content,
validate_options=False,
)
except DirectiveParsingError as err:
raise MystMetadataParsingError(
"{0} cell {1} at line {2} could not be read: {3}".format(
cell_type, cell_index, token.map[0] + 1, err
)
)
return options, body_lines


def read_cell_metadata(token, cell_index):
"""Return cell metadata"""
metadata = {}
@@ -242,11 +256,8 @@ def myst_to_notebook(
NOTE: we assume here that all of these directives are at the top-level,
i.e. not nested in other directives.
"""
raise_if_myst_is_not_available()

# parse markdown file up to the block level (i.e. don't worry about inline text)
parser = default_parser("html", disable_syntax=["inline"])
tokens = parser.parse(text + "\n")
tokens = get_parser().parse(text + "\n")
lines = text.splitlines()
md_start_line = 0

@@ -334,7 +345,6 @@ def notebook_to_myst(
:param default_lexer: a lexer name to use for annotating code cells
(if ``nb.metadata.language_info.pygments_lexer`` is not available)
"""
raise_if_myst_is_not_available()
string = ""

nb_metadata = from_nbnode(nb.metadata)
4 changes: 1 addition & 3 deletions requirements-dev.txt
Original file line number Diff line number Diff line change
@@ -7,6 +7,4 @@ nbconvert
sphinx-gallery
setuptools
toml

# Python 2
pathlib
isort
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
markdown-it-py~=0.5.2
nbformat>=4.0.0
pyyaml
toml
9 changes: 4 additions & 5 deletions setup.py
Original file line number Diff line number Diff line change
@@ -52,10 +52,12 @@
),
],
entry_points={"console_scripts": ["jupytext = jupytext.cli:jupytext"]},
python_requires=">=3.6",
tests_require=["pytest"],
install_requires=["nbformat>=4.0.0", "pyyaml", "toml", 'mock;python_version<"3"'],
install_requires=["markdown-it-py~=0.5.2", "nbformat>=4.0.0", "pyyaml", "toml"],
extras_require={
"myst": ["myst-parser~=0.8.0; python_version >= '3.6'"],
# left for back-compatibility
"myst": [],
"toml": ["toml"],
},
license="MIT",
@@ -68,10 +70,7 @@
"Intended Audience :: Science/Research",
"Topic :: Text Processing :: Markup",
"Programming Language :: Python",
"Programming Language :: Python :: 2.7",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
3 changes: 0 additions & 3 deletions tests/test_cli.py
Original file line number Diff line number Diff line change
@@ -27,7 +27,6 @@
from .utils import (
requires_jupytext_installed,
requires_pandoc,
requires_myst,
skip_on_windows,
)

@@ -979,7 +978,6 @@ def test_set_format_with_subfolder(tmpdir):
)


@requires_myst
@requires_pandoc
@pytest.mark.parametrize("format_name", ["md", "md:myst", "md:pandoc"])
def test_create_header_with_set_formats(format_name, tmpdir):
@@ -996,7 +994,6 @@ def test_create_header_with_set_formats(format_name, tmpdir):
assert nb["metadata"]["jupytext"]["formats"] == format_name


@requires_myst
@requires_pandoc
@pytest.mark.parametrize(
"format_name", ["md", "md:myst", "md:pandoc", "py:light", "py:percent"]
3 changes: 1 addition & 2 deletions tests/test_formats.py
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@
validate_one_format,
JupytextFormatError,
)
from .utils import list_notebooks, requires_myst, requires_pandoc
from .utils import list_notebooks, requires_pandoc


@pytest.mark.parametrize("nb_file", list_notebooks("python"))
@@ -328,7 +328,6 @@ def test_pandoc_format_is_preserved():
compare(formats_new, formats_org)


@requires_myst
def test_write_as_myst(tmpdir):
"""Inspired by https://github.com/mwouts/jupytext/issues/462"""
nb = new_notebook()
Loading