From 320f9b16160ab70299460e77200a34f851bc7db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 16 Sep 2021 01:26:13 -0400 Subject: [PATCH 01/15] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Support=20confi?= =?UTF-8?q?guration=20through=20docutils.conf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is part of GH-420, an effort to make myst_parser integrate better with Docutils. * myst_parser/docutils_.py (Parser): Use new _myst_docutils_setting_tuples function to generate a settings_spec. (Parser.parse): Use new create_myst_config function to reconstruct a MdParserConfig object from a Docutils configuration object. --- myst_parser/docutils_.py | 130 ++++++++++++++++++++++++++++++++++----- 1 file changed, 113 insertions(+), 17 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 1e9562c8..ea674460 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -3,9 +3,10 @@ .. include:: :parser: myst_parser.docutils_ """ -from typing import Tuple +from typing import Iterable, Tuple, Union -from docutils import nodes +from attr import Attribute +from docutils import nodes, frontend from docutils.parsers.rst import Parser as RstParser from markdown_it.token import Token from markdown_it.utils import AttrDict @@ -13,25 +14,117 @@ from myst_parser.main import MdParserConfig, default_parser -class Parser(RstParser): - """Docutils parser for Markedly Structured Text (MyST).""" +def validate_int( + setting, value, option_parser, config_parser=None, config_section=None +): + return int(value) + + +def validate_tuple(length: int): + def validate( + setting, value, option_parser, config_parser=None, config_section=None + ): + l = frontend.validate_comma_separated_list( + setting, value, option_parser, config_parser, config_section + ) + if len(l) != length: + MSG = "Expecting {} items in {}, got {}: {}." + raise ValueError(MSG.format(length, setting, len(l))) + return tuple(l) + + return validate + + +DOCUTILS_UNSET = object() +"""Sentinel for arguments not set through docutils.conf.""" + +DOCUTILS_OPTPARSE_OVERRIDES = { + "url_schemes": {"validator": frontend.validate_comma_separated_list}, + # url_schemes accepts an iterable or ``None``, but ``None`` is the same as ``()``. +} +"""Custom optparse configurations for docutils.conf entries.""" + +DOCUTILS_EXCLUDED_ARGS = ( + # docutils.conf can't represent callables + "heading_slug_func", + # docutils.conf can't represent dicts + "html_meta", + "substitutions", + # We don't want to set the renderer from docutils.conf + "renderer", +) +"""Names of settings that cannot be set in docutils.conf.""" - supported: Tuple[str, ...] = ("md", "markdown", "myst") - """Aliases this parser supports.""" - settings_spec = RstParser.settings_spec - """Runtime settings specification. +def _docutils_optparse_options_of_attribute(a: Attribute): + override = DOCUTILS_OPTPARSE_OVERRIDES.get(a.name) + if override is not None: + return override + if a.type is int: + return {"validator": validate_int} + if a.type is bool: + return {"validator": frontend.validate_boolean} + if a.type is str: + return {} + if a.type == Iterable[str]: + return {"validator": frontend.validate_comma_separated_list} + if a.type == Tuple[str, str]: + return {"validator": validate_tuple(2)} + if a.type == Union[int, type(None)] and a.default is None: + return {"validator": validate_int, "default": None} + if a.type == Union[Iterable[str], type(None)] and a.default is None: + return {"validator": frontend.validate_comma_separated_list, "default": None} + raise AssertionError( + f"""Configuration option {a.name} not set up for use \ +in docutils.conf. Either add {a.name} to docutils_.DOCUTILS_EXCLUDED_ARGS, or \ +add a new entry in _docutils_optparse_of_attribute.""" + ) - Defines runtime settings and associated command-line options, as used by - `docutils.frontend.OptionParser`. This is a concatenation of tuples of: - - Option group title (string or `None` which implies no group, just a list - of single options). +def _docutils_setting_tuple_of_attribute(attribute): + """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple.""" + name = "myst_" + attribute.name + flag = "--" + name.replace("_", "-") + options = {"dest": name, "default": DOCUTILS_UNSET} + options.update(_docutils_optparse_options_of_attribute(attribute)) + return (None, [flag], options) - - Description (string or `None`). - - A sequence of option tuples - """ +def _myst_docutils_setting_tuples(): + return tuple( + _docutils_setting_tuple_of_attribute(a) + for a in MdParserConfig.get_fields() + if not a.name in DOCUTILS_EXCLUDED_ARGS + ) + + +def create_myst_config(settings: frontend.Values): + values = {} + for attribute in MdParserConfig.get_fields(): + if attribute.name in DOCUTILS_EXCLUDED_ARGS: + continue + setting = f"myst_{attribute.name}" + val = getattr(settings, setting) + delattr(settings, setting) + if val is not DOCUTILS_UNSET: + values[attribute.name] = val + values["renderer"] = "docutils" + return MdParserConfig(**values) + + +class Parser(RstParser): + """Docutils parser for Markedly Structured Text (MyST).""" + + supported: Tuple[str, ...] = ("md", "markdown", "myst") + """Aliases this parser supports.""" + + settings_spec = ( + *RstParser.settings_spec, + "MyST options", + None, + _myst_docutils_setting_tuples(), + ) + """Runtime settings specification.""" config_section = "myst parser" config_section_dependencies = ("parsers",) @@ -42,9 +135,12 @@ def parse(self, inputstring: str, document: nodes.document) -> None: :param inputstring: The source string to parse :param document: The root docutils node to add AST elements to - """ - config = MdParserConfig(renderer="docutils") + try: + config = create_myst_config(document.settings) + except (TypeError, ValueError) as error: + document.reporter.error(f"myst configuration invalid: {error.args[0]}") + config = MdParserConfig(renderer="docutils") parser = default_parser(config) parser.options["document"] = document env = AttrDict() From 46aa23ff812ec088e7b83969f5ada7a9b885458e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Pit-Claudel?= Date: Thu, 16 Sep 2021 23:18:55 -0400 Subject: [PATCH 02/15] =?UTF-8?q?=F0=9F=91=8C=20IMPROVE:=20Add=20a=20comma?= =?UTF-8?q?nd-line=20entry=20point=20in=20=5Fdocutils.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- myst_parser/docutils_.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index ea674460..17ebd40a 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -150,3 +150,14 @@ def parse(self, inputstring: str, document: nodes.document) -> None: # specified in the sphinx configuration tokens = [Token("front_matter", "", 0, content="{}", map=[0, 0])] + tokens parser.renderer.render(tokens, parser.options, env) + + +if __name__ == "__main__": + from docutils.core import publish_cmdline, default_description + + publish_cmdline( + parser=Parser(), + writer_name='html', + description="Generates (X)HTML documents from standalone MyST sources. " + + default_description, + ) From be40bf73bc75b7fb99a2b5661c3bf4b3bc345c22 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Sep 2021 03:51:55 +0000 Subject: [PATCH 03/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- myst_parser/docutils_.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 17ebd40a..f89bc6d6 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -6,7 +6,7 @@ from typing import Iterable, Tuple, Union from attr import Attribute -from docutils import nodes, frontend +from docutils import frontend, nodes from docutils.parsers.rst import Parser as RstParser from markdown_it.token import Token from markdown_it.utils import AttrDict @@ -153,11 +153,11 @@ def parse(self, inputstring: str, document: nodes.document) -> None: if __name__ == "__main__": - from docutils.core import publish_cmdline, default_description + from docutils.core import default_description, publish_cmdline publish_cmdline( parser=Parser(), - writer_name='html', + writer_name="html", description="Generates (X)HTML documents from standalone MyST sources. " + default_description, ) From 25c8e7fc99e9c0807195447cd6a9bac325570ddc Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 5 Dec 2021 02:20:33 +0100 Subject: [PATCH 04/15] Add CLI entry-points and help strings --- myst_parser/docutils_.py | 161 +++++++++++++++++++++---------- myst_parser/docutils_renderer.py | 4 +- myst_parser/main.py | 69 ++++++++++--- setup.cfg | 5 + tests/test_docutils.py | 2 +- 5 files changed, 177 insertions(+), 64 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 44173502..56343793 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -3,45 +3,45 @@ .. include:: :parser: myst_parser.docutils_ """ -from typing import Iterable, Tuple, Union +from typing import Any, Callable, Iterable, Tuple, Union from attr import Attribute from docutils import frontend, nodes +from docutils.core import default_description, publish_cmdline from docutils.parsers.rst import Parser as RstParser from markdown_it.token import Token from myst_parser.main import MdParserConfig, default_parser -def validate_int( +def _validate_int( setting, value, option_parser, config_parser=None, config_section=None -): +) -> int: + """Validate an integer setting.""" return int(value) -def validate_tuple(length: int): - def validate( +def _create_validate_tuple(length: int) -> Callable[..., Tuple[str, ...]]: + """Create a validator for a tuple of length `length`.""" + + def _validate( setting, value, option_parser, config_parser=None, config_section=None ): - l = frontend.validate_comma_separated_list( + string_list = frontend.validate_comma_separated_list( setting, value, option_parser, config_parser, config_section ) - if len(l) != length: - MSG = "Expecting {} items in {}, got {}: {}." - raise ValueError(MSG.format(length, setting, len(l))) - return tuple(l) + if len(string_list) != length: + raise ValueError( + f"Expecting {length} items in {setting}, got {len(string_list)}." + ) + return tuple(string_list) - return validate + return _validate DOCUTILS_UNSET = object() """Sentinel for arguments not set through docutils.conf.""" -DOCUTILS_OPTPARSE_OVERRIDES = { - "url_schemes": {"validator": frontend.validate_comma_separated_list}, - # url_schemes accepts an iterable or ``None``, but ``None`` is the same as ``()``. -} -"""Custom optparse configurations for docutils.conf entries.""" DOCUTILS_EXCLUDED_ARGS = ( # docutils.conf can't represent callables @@ -49,51 +49,70 @@ def validate( # docutils.conf can't represent dicts "html_meta", "substitutions", + # we can't add substitutions so not needed + "sub_delimiters", # We don't want to set the renderer from docutils.conf "renderer", ) """Names of settings that cannot be set in docutils.conf.""" -def _docutils_optparse_options_of_attribute(a: Attribute): - override = DOCUTILS_OPTPARSE_OVERRIDES.get(a.name) - if override is not None: - return override - if a.type is int: - return {"validator": validate_int} - if a.type is bool: - return {"validator": frontend.validate_boolean} - if a.type is str: - return {} - if a.type == Iterable[str]: - return {"validator": frontend.validate_comma_separated_list} - if a.type == Tuple[str, str]: - return {"validator": validate_tuple(2)} - if a.type == Union[int, type(None)] and a.default is None: - return {"validator": validate_int, "default": None} - if a.type == Union[Iterable[str], type(None)] and a.default is None: - return {"validator": frontend.validate_comma_separated_list, "default": None} +def _docutils_optparse_options_of_attribute( + at: Attribute, default: Any +) -> Tuple[dict, str]: + """Convert an ``MdParserConfig`` attribute into a Docutils optparse options dict.""" + if at.type is int: + return {"validator": _validate_int}, f"(type: int, default: {default})" + if at.type is bool: + return { + "validator": frontend.validate_boolean + }, f"(type: bool, default: {default})" + if at.type is str: + return {}, f"(type: str, default: '{default}')" + if at.type == Iterable[str] or at.name == "url_schemes": + return { + "validator": frontend.validate_comma_separated_list + }, f"(type: comma-delimited, default: '{','.join(default)}')" + if at.type == Tuple[str, str]: + return { + "validator": _create_validate_tuple(2) + }, f"(type: str,str, default: '{','.join(default)}')" + if at.type == Union[int, type(None)] and at.default is None: + return { + "validator": _validate_int, + "default": None, + }, f"(type: null|int, default: {default})" + if at.type == Union[Iterable[str], type(None)] and at.default is None: + return { + "validator": frontend.validate_comma_separated_list, + "default": None, + }, f"(type: comma-delimited, default: '{default or ','.join(default)}')" raise AssertionError( - f"""Configuration option {a.name} not set up for use \ -in docutils.conf. Either add {a.name} to docutils_.DOCUTILS_EXCLUDED_ARGS, or \ -add a new entry in _docutils_optparse_of_attribute.""" + f"Configuration option {at.name} not set up for use in docutils.conf." + f"Either add {at.name} to docutils_.DOCUTILS_EXCLUDED_ARGS," + "or add a new entry in _docutils_optparse_of_attribute." ) -def _docutils_setting_tuple_of_attribute(attribute): +def _docutils_setting_tuple_of_attribute( + attribute: Attribute, default: Any +) -> Tuple[str, Any, Any]: """Convert an ``MdParserConfig`` attribute into a Docutils setting tuple.""" - name = "myst_" + attribute.name + name = f"myst_{attribute.name}" flag = "--" + name.replace("_", "-") options = {"dest": name, "default": DOCUTILS_UNSET} - options.update(_docutils_optparse_options_of_attribute(attribute)) - return (None, [flag], options) + at_options, type_str = _docutils_optparse_options_of_attribute(attribute, default) + options.update(at_options) + help_str = attribute.metadata.get("help", "") if attribute.metadata else "" + return (f"{help_str} {type_str}", [flag], options) def _myst_docutils_setting_tuples(): + defaults = MdParserConfig() return tuple( - _docutils_setting_tuple_of_attribute(a) - for a in MdParserConfig.get_fields() - if not a.name in DOCUTILS_EXCLUDED_ARGS + _docutils_setting_tuple_of_attribute(at, getattr(defaults, at.name)) + for at in MdParserConfig.get_fields() + if at.name not in DOCUTILS_EXCLUDED_ARGS ) @@ -151,12 +170,56 @@ def parse(self, inputstring: str, document: nodes.document) -> None: parser.renderer.render(tokens, parser.options, env) -if __name__ == "__main__": - from docutils.core import default_description, publish_cmdline - +def cli_html(): + """Cmdline entrypoint for converting MyST to HTML.""" publish_cmdline( parser=Parser(), writer_name="html", - description="Generates (X)HTML documents from standalone MyST sources. " - + default_description, + description=( + f"Generates (X)HTML documents from standalone MyST sources.\n{default_description}" + ), + ) + + +def cli_html5(): + """Cmdline entrypoint for converting MyST to HTML5.""" + publish_cmdline( + parser=Parser(), + writer_name="html5", + description=( + f"Generates HTML5 documents from standalone MyST sources.\n{default_description}" + ), + ) + + +def cli_latex(): + """Cmdline entrypoint for converting MyST to LaTeX.""" + publish_cmdline( + parser=Parser(), + writer_name="latex", + description=( + f"Generates LaTeX documents from standalone MyST sources.\n{default_description}" + ), + ) + + +def cli_xml(): + """Cmdline entrypoint for converting MyST to XML.""" + publish_cmdline( + parser=Parser(), + writer_name="xml", + description=( + f"Generates Docutils-native XML from standalone MyST sources.\n{default_description}" + ), + ) + + +def cli_pseudoxml(): + """Cmdline entrypoint for converting MyST to pseudo-XML.""" + publish_cmdline( + parser=Parser(), + writer_name="pseudoxml", + description=( + f"Generates pseudo-XML from standalone MyST sources.\n{default_description}" + ), ) diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py index 76c6617c..a7b63a59 100644 --- a/myst_parser/docutils_renderer.py +++ b/myst_parser/docutils_renderer.py @@ -52,9 +52,9 @@ from .utils import is_external_url -def make_document(source_path="notset") -> nodes.document: +def make_document(source_path="notset", parser_cls=RSTParser) -> nodes.document: """Create a new docutils document.""" - settings = OptionParser(components=(RSTParser,)).get_default_values() + settings = OptionParser(components=(parser_cls,)).get_default_values() return new_document(source_path, settings=settings) diff --git a/myst_parser/main.py b/myst_parser/main.py index 07376cf8..eef96853 100644 --- a/myst_parser/main.py +++ b/myst_parser/main.py @@ -37,19 +37,46 @@ class MdParserConfig: renderer: str = attr.ib( default="sphinx", validator=in_(["sphinx", "html", "docutils"]) ) - commonmark_only: bool = attr.ib(default=False, validator=instance_of(bool)) - enable_extensions: Iterable[str] = attr.ib(factory=lambda: ["dollarmath"]) + commonmark_only: bool = attr.ib( + default=False, + validator=instance_of(bool), + metadata={"help": "Use strict CommonMark parser"}, + ) + enable_extensions: Iterable[str] = attr.ib( + factory=lambda: ["dollarmath"], metadata={"help": "Enable extensions"} + ) - dmath_allow_labels: bool = attr.ib(default=True, validator=instance_of(bool)) - dmath_allow_space: bool = attr.ib(default=True, validator=instance_of(bool)) - dmath_allow_digits: bool = attr.ib(default=True, validator=instance_of(bool)) - dmath_double_inline: bool = attr.ib(default=False, validator=instance_of(bool)) + dmath_allow_labels: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "Parse `$$...$$ (label)`"}, + ) + dmath_allow_space: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "Allow intial/final spaces in `$ ... $`"}, + ) + dmath_allow_digits: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "Allow digits in `1$ ...$2`"}, + ) + dmath_double_inline: bool = attr.ib( + default=False, + validator=instance_of(bool), + metadata={"help": "Parse inline `$$ ... $$`"}, + ) - update_mathjax: bool = attr.ib(default=True, validator=instance_of(bool)) + update_mathjax: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "Update mathjax configuration"}, + ) mathjax_classes: str = attr.ib( default="tex2jax_process|mathjax_process|math|output_area", validator=instance_of(str), + metadata={"help": "MathJax classes to add to HTML"}, ) @enable_extensions.validator @@ -77,29 +104,40 @@ def check_extensions(self, attribute, value): disable_syntax: Iterable[str] = attr.ib( factory=list, validator=deep_iterable(instance_of(str), instance_of((list, tuple))), + metadata={"help": "Disable syntax elements"}, ) # see https://en.wikipedia.org/wiki/List_of_URI_schemes url_schemes: Optional[Iterable[str]] = attr.ib( default=cast(Optional[Iterable[str]], ("http", "https", "mailto", "ftp")), validator=optional(deep_iterable(instance_of(str), instance_of((list, tuple)))), + metadata={"help": "URL schemes to allow in links"}, ) heading_anchors: Optional[int] = attr.ib( - default=None, validator=optional(in_([1, 2, 3, 4, 5, 6, 7])) + default=None, + validator=optional(in_([1, 2, 3, 4, 5, 6, 7])), + metadata={"help": "Heading level depth to assign anchors"}, ) heading_slug_func: Optional[Callable[[str], str]] = attr.ib( - default=None, validator=optional(is_callable()) + default=None, + validator=optional(is_callable()), + metadata={"help": "Function for creating heading anchors"}, ) html_meta: Dict[str, str] = attr.ib( factory=dict, validator=deep_mapping(instance_of(str), instance_of(str), instance_of(dict)), repr=lambda v: str(list(v)), + metadata={"help": "HTML meta tags"}, ) - footnote_transition: bool = attr.ib(default=True, validator=instance_of(bool)) + footnote_transition: bool = attr.ib( + default=True, + validator=instance_of(bool), + metadata={"help": "Place a transition before any footnotes"}, + ) substitutions: Dict[str, Union[str, int, float]] = attr.ib( factory=dict, @@ -107,11 +145,18 @@ def check_extensions(self, attribute, value): instance_of(str), instance_of((str, int, float)), instance_of(dict) ), repr=lambda v: str(list(v)), + metadata={"help": "Substitutions"}, ) - sub_delimiters: Tuple[str, str] = attr.ib(default=("{", "}")) + sub_delimiters: Tuple[str, str] = attr.ib( + default=("{", "}"), metadata={"help": "Substitution delimiters"} + ) - words_per_minute: int = attr.ib(default=200, validator=instance_of(int)) + words_per_minute: int = attr.ib( + default=200, + validator=instance_of(int), + metadata={"help": "For reading speed calculations"}, + ) @sub_delimiters.validator def check_sub_delimiters(self, attribute, value): diff --git a/setup.cfg b/setup.cfg index 7c440a98..4f7ddcbe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -50,6 +50,11 @@ zip_safe = True [options.entry_points] console_scripts = myst-anchors = myst_parser.cli:print_anchors + myst-docutils-html = myst_parser.docutils_:cli_html + myst-docutils-html5 = myst_parser.docutils_:cli_html5 + myst-docutils-latex = myst_parser.docutils_:cli_latex + myst-docutils-xml = myst_parser.docutils_:cli_xml + myst-docutils-pseudoxml = myst_parser.docutils_:cli_pseudoxml [options.extras_require] code_style = diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 4cbb4fc9..f365385a 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -4,7 +4,7 @@ def test_parser(): parser = Parser() - document = make_document() + document = make_document(parser_cls=Parser) parser.parse("something", document) assert ( document.pformat().strip() From 49000098c0b2b68553756c31e649e99f6f4b226e Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 5 Dec 2021 03:01:51 +0100 Subject: [PATCH 05/15] improve --- myst_parser/docutils_.py | 5 +++++ myst_parser/docutils_renderer.py | 4 ++-- myst_parser/main.py | 12 ++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 56343793..2c60a1d0 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -51,6 +51,11 @@ def _validate( "substitutions", # we can't add substitutions so not needed "sub_delimiters", + # heading anchors are currently sphinx only + "heading_anchors", + # sphinx.ext.mathjax only options + "update_mathjax", + "mathjax_classes", # We don't want to set the renderer from docutils.conf "renderer", ) diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py index a7b63a59..3c3708ae 100644 --- a/myst_parser/docutils_renderer.py +++ b/myst_parser/docutils_renderer.py @@ -495,8 +495,8 @@ def render_heading(self, token: SyntaxTreeNode) -> None: new_section = nodes.section() if level == 1 and ( - self.sphinx_env is None - or ( + self.sphinx_env is not None + and ( "myst_update_mathjax" in self.sphinx_env.config and self.sphinx_env.config.myst_update_mathjax ) diff --git a/myst_parser/main.py b/myst_parser/main.py index eef96853..38153e6b 100644 --- a/myst_parser/main.py +++ b/myst_parser/main.py @@ -54,29 +54,29 @@ class MdParserConfig: dmath_allow_space: bool = attr.ib( default=True, validator=instance_of(bool), - metadata={"help": "Allow intial/final spaces in `$ ... $`"}, + metadata={"help": "dollarmath: allow initial/final spaces in `$ ... $`"}, ) dmath_allow_digits: bool = attr.ib( default=True, validator=instance_of(bool), - metadata={"help": "Allow digits in `1$ ...$2`"}, + metadata={"help": "dollarmath: allow initial/final digits `1$ ...$2`"}, ) dmath_double_inline: bool = attr.ib( default=False, validator=instance_of(bool), - metadata={"help": "Parse inline `$$ ... $$`"}, + metadata={"help": "dollarmath: parse inline `$$ ... $$`"}, ) update_mathjax: bool = attr.ib( default=True, validator=instance_of(bool), - metadata={"help": "Update mathjax configuration"}, + metadata={"help": "Update sphinx.ext.mathjax configuration"}, ) mathjax_classes: str = attr.ib( default="tex2jax_process|mathjax_process|math|output_area", validator=instance_of(str), - metadata={"help": "MathJax classes to add to HTML"}, + metadata={"help": "MathJax classes to add to math HTML"}, ) @enable_extensions.validator @@ -117,7 +117,7 @@ def check_extensions(self, attribute, value): heading_anchors: Optional[int] = attr.ib( default=None, validator=optional(in_([1, 2, 3, 4, 5, 6, 7])), - metadata={"help": "Heading level depth to assign anchors"}, + metadata={"help": "Heading level depth to assign HTML anchors"}, ) heading_slug_func: Optional[Callable[[str], str]] = attr.ib( From fee3f884168a9069ae9b72a8617f79e09f600991 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 5 Dec 2021 03:09:43 +0100 Subject: [PATCH 06/15] Update docutils_renderer.py --- myst_parser/docutils_renderer.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py index 3c3708ae..79200bba 100644 --- a/myst_parser/docutils_renderer.py +++ b/myst_parser/docutils_renderer.py @@ -474,6 +474,15 @@ def render_fence(self, token: SyntaxTreeNode) -> None: self.add_line_and_source_path(node, token) self.current_node.append(node) + @property + def blocks_mathjax_processing(self) -> bool: + """Only add mathjax ignore classes if using sphinx and myst_update_mathjax is True.""" + return ( + self.sphinx_env is not None + and "myst_update_mathjax" in self.sphinx_env.config + and self.sphinx_env.config.myst_update_mathjax + ) + def render_heading(self, token: SyntaxTreeNode) -> None: if self.md_env.get("match_titles", None) is False: @@ -494,13 +503,7 @@ def render_heading(self, token: SyntaxTreeNode) -> None: self.add_line_and_source_path(title_node, token) new_section = nodes.section() - if level == 1 and ( - self.sphinx_env is not None - and ( - "myst_update_mathjax" in self.sphinx_env.config - and self.sphinx_env.config.myst_update_mathjax - ) - ): + if level == 1 and self.blocks_mathjax_processing: new_section["classes"].extend(["tex2jax_ignore", "mathjax_ignore"]) self.add_line_and_source_path(new_section, token) new_section.append(title_node) From 015c452d264677e654bd78a9804a8fcb87427226 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 5 Dec 2021 09:01:45 +0100 Subject: [PATCH 07/15] Add tests --- myst_parser/docutils_renderer.py | 2 +- tests/test_docutils.py | 57 +++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/myst_parser/docutils_renderer.py b/myst_parser/docutils_renderer.py index 79200bba..96ae3485 100644 --- a/myst_parser/docutils_renderer.py +++ b/myst_parser/docutils_renderer.py @@ -53,7 +53,7 @@ def make_document(source_path="notset", parser_cls=RSTParser) -> nodes.document: - """Create a new docutils document.""" + """Create a new docutils document, with the parser classes' default settings.""" settings = OptionParser(components=(parser_cls,)).get_default_values() return new_document(source_path, settings=settings) diff --git a/tests/test_docutils.py b/tests/test_docutils.py index f365385a..6301c9be 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -1,8 +1,18 @@ -from myst_parser.docutils_ import Parser +import io + +from myst_parser.docutils_ import ( + Parser, + cli_html, + cli_html5, + cli_latex, + cli_pseudoxml, + cli_xml, +) from myst_parser.docutils_renderer import make_document def test_parser(): + """Test calling `Parser.parse` directly.""" parser = Parser() document = make_document(parser_cls=Parser) parser.parse("something", document) @@ -10,3 +20,48 @@ def test_parser(): document.pformat().strip() == '\n \n something' ) + + +def test_cli_html(monkeypatch, capsys): + monkeypatch.setattr("sys.argv", ["cli"]) + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_html() + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out + + +def test_cli_html5(monkeypatch, capsys): + monkeypatch.setattr("sys.argv", ["cli"]) + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_html5() + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out + + +def test_cli_latex(monkeypatch, capsys): + monkeypatch.setattr("sys.argv", ["cli"]) + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_latex() + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out + + +def test_cli_xml(monkeypatch, capsys): + monkeypatch.setattr("sys.argv", ["cli"]) + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_xml() + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out + + +def test_cli_pseudoxml(monkeypatch, capsys): + monkeypatch.setattr("sys.argv", ["cli"]) + monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) + cli_pseudoxml() + captured = capsys.readouterr() + assert not captured.err + assert "text" in captured.out From 6462de4b6768c7451cac09d5552187230be6dc2b Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 5 Dec 2021 09:49:39 +0100 Subject: [PATCH 08/15] improve --- myst_parser/docutils_.py | 58 +++++++++++++++------------------------- tests/test_docutils.py | 24 ++++++++++------- 2 files changed, 35 insertions(+), 47 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 2c60a1d0..47d87c0a 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -3,7 +3,7 @@ .. include:: :parser: myst_parser.docutils_ """ -from typing import Any, Callable, Iterable, Tuple, Union +from typing import Any, Callable, Iterable, List, Optional, Tuple, Union from attr import Attribute from docutils import frontend, nodes @@ -113,6 +113,7 @@ def _docutils_setting_tuple_of_attribute( def _myst_docutils_setting_tuples(): + """Return a list of Docutils setting for the MyST section.""" defaults = MdParserConfig() return tuple( _docutils_setting_tuple_of_attribute(at, getattr(defaults, at.name)) @@ -122,6 +123,7 @@ def _myst_docutils_setting_tuples(): def create_myst_config(settings: frontend.Values): + """Create a ``MdParserConfig`` from the given settings.""" values = {} for attribute in MdParserConfig.get_fields(): if attribute.name in DOCUTILS_EXCLUDED_ARGS: @@ -175,56 +177,38 @@ def parse(self, inputstring: str, document: nodes.document) -> None: parser.renderer.render(tokens, parser.options, env) -def cli_html(): - """Cmdline entrypoint for converting MyST to HTML.""" +def _run_cli(writer_name: str, writer_description: str, argv: Optional[List[str]]): + """Run the command line interface for a particular writer.""" publish_cmdline( parser=Parser(), - writer_name="html", + writer_name=writer_name, description=( - f"Generates (X)HTML documents from standalone MyST sources.\n{default_description}" + f"Generates {writer_description} from standalone MyST sources.\n{default_description}" ), + argv=argv, ) -def cli_html5(): +def cli_html(argv: Optional[List[str]] = None) -> None: + """Cmdline entrypoint for converting MyST to HTML.""" + _run_cli("html", "(X)HTML documents", argv) + + +def cli_html5(argv: Optional[List[str]] = None): """Cmdline entrypoint for converting MyST to HTML5.""" - publish_cmdline( - parser=Parser(), - writer_name="html5", - description=( - f"Generates HTML5 documents from standalone MyST sources.\n{default_description}" - ), - ) + _run_cli("html5", "HTML5 documents", argv) -def cli_latex(): +def cli_latex(argv: Optional[List[str]] = None): """Cmdline entrypoint for converting MyST to LaTeX.""" - publish_cmdline( - parser=Parser(), - writer_name="latex", - description=( - f"Generates LaTeX documents from standalone MyST sources.\n{default_description}" - ), - ) + _run_cli("latex", "LaTeX documents", argv) -def cli_xml(): +def cli_xml(argv: Optional[List[str]] = None): """Cmdline entrypoint for converting MyST to XML.""" - publish_cmdline( - parser=Parser(), - writer_name="xml", - description=( - f"Generates Docutils-native XML from standalone MyST sources.\n{default_description}" - ), - ) + _run_cli("xml", "Docutils-native XML", argv) -def cli_pseudoxml(): +def cli_pseudoxml(argv: Optional[List[str]] = None): """Cmdline entrypoint for converting MyST to pseudo-XML.""" - publish_cmdline( - parser=Parser(), - writer_name="pseudoxml", - description=( - f"Generates pseudo-XML from standalone MyST sources.\n{default_description}" - ), - ) + _run_cli("pseudoxml", "pseudo-XML", argv) diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 6301c9be..dd58dc42 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -23,45 +23,49 @@ def test_parser(): def test_cli_html(monkeypatch, capsys): - monkeypatch.setattr("sys.argv", ["cli"]) monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) - cli_html() + cli_html([]) captured = capsys.readouterr() assert not captured.err assert "text" in captured.out def test_cli_html5(monkeypatch, capsys): - monkeypatch.setattr("sys.argv", ["cli"]) monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) - cli_html5() + cli_html5([]) captured = capsys.readouterr() assert not captured.err assert "text" in captured.out def test_cli_latex(monkeypatch, capsys): - monkeypatch.setattr("sys.argv", ["cli"]) monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) - cli_latex() + cli_latex([]) captured = capsys.readouterr() assert not captured.err assert "text" in captured.out def test_cli_xml(monkeypatch, capsys): - monkeypatch.setattr("sys.argv", ["cli"]) monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) - cli_xml() + cli_xml([]) captured = capsys.readouterr() assert not captured.err assert "text" in captured.out def test_cli_pseudoxml(monkeypatch, capsys): - monkeypatch.setattr("sys.argv", ["cli"]) monkeypatch.setattr("sys.stdin", io.TextIOWrapper(io.BytesIO(b"text"))) - cli_pseudoxml() + cli_pseudoxml([]) captured = capsys.readouterr() assert not captured.err assert "text" in captured.out + + +def test_help_text(): + """Test retrieving settings help text.""" + from docutils.frontend import OptionParser + + stream = io.StringIO() + OptionParser(components=(Parser,)).print_help(stream) + assert "MyST options" in stream.getvalue() From db79afc540f0125ce54c6c5dff7a18671aa951af Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 5 Dec 2021 10:05:10 +0100 Subject: [PATCH 09/15] update --- myst_parser/docutils_.py | 8 +++++--- myst_parser/sphinx_.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 47d87c0a..fd311c1b 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -1,8 +1,9 @@ """A module for compatibility with the docutils>=0.17 `include` directive, in RST documents:: - .. include:: + .. include:: path/to/file.md :parser: myst_parser.docutils_ """ +from contextlib import suppress from typing import Any, Callable, Iterable, List, Optional, Tuple, Union from attr import Attribute @@ -129,8 +130,9 @@ def create_myst_config(settings: frontend.Values): if attribute.name in DOCUTILS_EXCLUDED_ARGS: continue setting = f"myst_{attribute.name}" - val = getattr(settings, setting) - delattr(settings, setting) + val = getattr(settings, setting, DOCUTILS_UNSET) + with suppress(AttributeError): + delattr(settings, setting) if val is not DOCUTILS_UNSET: values[attribute.name] = val values["renderer"] = "docutils" diff --git a/myst_parser/sphinx_.py b/myst_parser/sphinx_.py index e59ae62a..e7979c80 100644 --- a/myst_parser/sphinx_.py +++ b/myst_parser/sphinx_.py @@ -1,6 +1,6 @@ """A module for compatibility with the docutils>=0.17 `include` directive, in RST documents:: - .. include:: + .. include:: path/to/file.md :parser: myst_parser.sphinx_ """ from myst_parser.sphinx_parser import MystParser as Parser # noqa: F401 From 3e3c2e207b05e3d07ac97c49adce4d3cd9722027 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Sun, 5 Dec 2021 10:19:20 +0100 Subject: [PATCH 10/15] Update test_docutils.py --- tests/test_docutils.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/test_docutils.py b/tests/test_docutils.py index dd58dc42..dea2e9c7 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -1,4 +1,5 @@ import io +from textwrap import dedent from myst_parser.docutils_ import ( Parser, @@ -69,3 +70,28 @@ def test_help_text(): stream = io.StringIO() OptionParser(components=(Parser,)).print_help(stream) assert "MyST options" in stream.getvalue() + + +def test_include_from_rst(tmp_path): + """Test including a MyST file from within an RST file.""" + from docutils.parsers.rst import Parser as RSTParser + + include_path = tmp_path.joinpath("include.md") + include_path.write_text("# Title") + + parser = RSTParser() + document = make_document(parser_cls=RSTParser) + parser.parse( + f".. include:: {include_path}\n :parser: myst_parser.docutils_", document + ) + assert ( + document.pformat().strip() + == dedent( + """\ + +
+ + Title + """ + ).strip() + ) From 8ffb92f6fe7c7439b65d8e42c155372f54a0d87c Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Sun, 5 Dec 2021 10:29:44 +0100 Subject: [PATCH 11/15] Update test_docutils.py --- tests/test_docutils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_docutils.py b/tests/test_docutils.py index dea2e9c7..8432b863 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -1,6 +1,9 @@ import io from textwrap import dedent +import pytest +from docutils import VersionInfo, __version_info__ + from myst_parser.docutils_ import ( Parser, cli_html, @@ -72,6 +75,9 @@ def test_help_text(): assert "MyST options" in stream.getvalue() +@pytest.mark.skipif( + __version_info__ < VersionInfo(0, 17), reason="parser option added in docutils 0.17" +) def test_include_from_rst(tmp_path): """Test including a MyST file from within an RST file.""" from docutils.parsers.rst import Parser as RSTParser From 90a4e8fbdcbd54382ddd7dbb5511eb1e4616d1a0 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Sun, 5 Dec 2021 10:33:48 +0100 Subject: [PATCH 12/15] Update test_docutils.py --- tests/test_docutils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_docutils.py b/tests/test_docutils.py index 8432b863..52d32810 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -76,7 +76,8 @@ def test_help_text(): @pytest.mark.skipif( - __version_info__ < VersionInfo(0, 17), reason="parser option added in docutils 0.17" + __version_info__ < VersionInfo(0, 17, 0, "final", 0, True), + reason="parser option added in docutils 0.17", ) def test_include_from_rst(tmp_path): """Test including a MyST file from within an RST file.""" From 30ba7d76a48e8195966a7cdcad4661cb7a3b685e Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Sun, 5 Dec 2021 11:37:42 +0100 Subject: [PATCH 13/15] Add sphinx-build test for including myst in rst --- .../sourcedirs/include_from_rst/conf.py | 2 ++ .../sourcedirs/include_from_rst/include.md | 5 +++ .../sourcedirs/include_from_rst/index.rst | 5 +++ tests/test_sphinx/test_sphinx_builds.py | 33 +++++++++++++++++++ .../test_include_from_rst.xml | 10 ++++++ 5 files changed, 55 insertions(+) create mode 100644 tests/test_sphinx/sourcedirs/include_from_rst/conf.py create mode 100644 tests/test_sphinx/sourcedirs/include_from_rst/include.md create mode 100644 tests/test_sphinx/sourcedirs/include_from_rst/index.rst create mode 100644 tests/test_sphinx/test_sphinx_builds/test_include_from_rst.xml diff --git a/tests/test_sphinx/sourcedirs/include_from_rst/conf.py b/tests/test_sphinx/sourcedirs/include_from_rst/conf.py new file mode 100644 index 00000000..a743e3c9 --- /dev/null +++ b/tests/test_sphinx/sourcedirs/include_from_rst/conf.py @@ -0,0 +1,2 @@ +extensions = ["myst_parser"] +exclude_patterns = ["_build", "include.md"] diff --git a/tests/test_sphinx/sourcedirs/include_from_rst/include.md b/tests/test_sphinx/sourcedirs/include_from_rst/include.md new file mode 100644 index 00000000..20f34b9c --- /dev/null +++ b/tests/test_sphinx/sourcedirs/include_from_rst/include.md @@ -0,0 +1,5 @@ +# Markdown + +[target] + +[target]: http://example.com/ diff --git a/tests/test_sphinx/sourcedirs/include_from_rst/index.rst b/tests/test_sphinx/sourcedirs/include_from_rst/index.rst new file mode 100644 index 00000000..0bb31777 --- /dev/null +++ b/tests/test_sphinx/sourcedirs/include_from_rst/index.rst @@ -0,0 +1,5 @@ +Title +===== + +.. include:: include.md + :parser: myst_parser.sphinx_ diff --git a/tests/test_sphinx/test_sphinx_builds.py b/tests/test_sphinx/test_sphinx_builds.py index 3470487c..c7d8a9e0 100644 --- a/tests/test_sphinx/test_sphinx_builds.py +++ b/tests/test_sphinx/test_sphinx_builds.py @@ -12,6 +12,7 @@ import pytest import sphinx +from docutils import VersionInfo, __version_info__ SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "sourcedirs")) @@ -269,6 +270,38 @@ def test_includes( ) +@pytest.mark.skipif( + __version_info__ < VersionInfo(0, 17, 0, "final", 0, True), + reason="parser option added in docutils 0.17", +) +@pytest.mark.sphinx( + buildername="html", + srcdir=os.path.join(SOURCE_DIR, "include_from_rst"), + freshenv=True, +) +def test_include_from_rst( + app, + status, + warning, + get_sphinx_app_doctree, + get_sphinx_app_output, + remove_sphinx_builds, +): + """Test of include directive inside RST file.""" + app.build() + + assert "build succeeded" in status.getvalue() # Build succeeded + warnings = warning.getvalue().strip() + assert warnings == "" + + get_sphinx_app_doctree( + app, + docname="index", + regress=True, + regress_ext=".xml", + ) + + @pytest.mark.sphinx( buildername="html", srcdir=os.path.join(SOURCE_DIR, "footnotes"), freshenv=True ) diff --git a/tests/test_sphinx/test_sphinx_builds/test_include_from_rst.xml b/tests/test_sphinx/test_sphinx_builds/test_include_from_rst.xml new file mode 100644 index 00000000..6f325067 --- /dev/null +++ b/tests/test_sphinx/test_sphinx_builds/test_include_from_rst.xml @@ -0,0 +1,10 @@ +<document source="index.rst"> + <section ids="title" names="title"> + <title> + Title + <section classes="tex2jax_ignore mathjax_ignore" ids="markdown" names="markdown"> + <title> + Markdown + <paragraph> + <reference refuri="http://example.com/"> + target From 34322ce52ca746201f600020ab416d5986eb7855 Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Sun, 5 Dec 2021 16:06:57 +0100 Subject: [PATCH 14/15] Add documentation --- docs/api/parsers.md | 25 ++++++++++--- docs/api/reference.rst | 10 +++++ docs/conf.py | 32 +++++++++++++++- docs/docutils.md | 79 ++++++++++++++++++++++++++++++++++++++++ docs/index.md | 1 + docs/sphinx/index.md | 2 + docs/sphinx/use.md | 14 +++++++ myst_parser/docutils_.py | 9 ++++- myst_parser/main.py | 4 ++ 9 files changed, 169 insertions(+), 7 deletions(-) create mode 100644 docs/docutils.md diff --git a/docs/api/parsers.md b/docs/api/parsers.md index 581dc563..7b499844 100644 --- a/docs/api/parsers.md +++ b/docs/api/parsers.md @@ -9,28 +9,34 @@ The MyST Parser comes bundled with some helper functions to quickly parse MyST Markdown and render its output. +:::{important} +These APIs are primarily intended for testing and development purposes. +For proper parsing see {ref}`myst-sphinx` and {ref}`myst-docutils`. +::: + ### Parse MyST Markdown to HTML -For example, the following code parses markdown and renders as HTML: +The following code parses markdown and renders as HTML using only the markdown-it parser +(i.e. no sphinx or docutils specific processing is done): ```python from myst_parser.main import to_html -to_html("some *text*") +to_html("some *text* {literal}`a`") ``` <!-- #region --> ```html -'<p>some <em>text</em></p>\n' +'<p>some <em>text</em> <code class="myst role">{literal}[a]</code></p>\n' ``` <!-- #endregion --> ### Parse MyST Markdown to docutils -The following function renders your text as **docutils objects** (for example, for use with the Sphinx ecosystem): +The following function renders your text as **docutils AST objects** (for example, for use with the Sphinx ecosystem): ```python from myst_parser.main import to_docutils -print(to_docutils("some *text*").pformat()) +print(to_docutils("some *text* {literal}`a`").pformat()) ``` ```xml @@ -39,8 +45,17 @@ print(to_docutils("some *text*").pformat()) some <emphasis> text + + <literal> + a ``` +:::{note} +This function only performs the initial parse of the AST, +without applying any transforms or post-processing. +See for example the [Sphinx core events](https://www.sphinx-doc.org/en/master/extdev/appapi.html?highlight=config-inited#sphinx-core-events). +::: + ### Parse MyST Markdown as `markdown-it` tokens The MyST Parser uses `markdown-it-py` tokens as an intermediate representation of your text. diff --git a/docs/api/reference.rst b/docs/api/reference.rst index 9fde1952..ad395cd6 100644 --- a/docs/api/reference.rst +++ b/docs/api/reference.rst @@ -81,6 +81,16 @@ Additional Methods .. autofunction:: myst_parser.sphinx_renderer.mock_sphinx_env +.. _api/docutils_parser: + +Docutils Parser Reference +------------------------- + +.. autoclass:: myst_parser.docutils_.Parser + :members: parse + :undoc-members: + :member-order: bysource + :show-inheritance: .. _api/sphinx_parser: diff --git a/docs/conf.py b/docs/conf.py index cd3f4bc6..6fce4b9e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -4,6 +4,9 @@ # list see the documentation: # https://www.sphinx-doc.org/en/master/usage/configuration.html +from sphinx.application import Sphinx +from sphinx.util.docutils import SphinxDirective + from myst_parser import __version__ # -- Project information ----------------------------------------------------- @@ -166,7 +169,34 @@ def run_apidoc(app): ] -def setup(app): +def setup(app: Sphinx): """Add functions to the Sphinx setup.""" + + class DocutilsCliHelpDirective(SphinxDirective): + """Directive to print the docutils CLI help.""" + + has_content = False + required_arguments = 0 + optional_arguments = 0 + final_argument_whitespace = False + option_spec = {} + + def run(self): + """Run the directive.""" + import io + + from docutils import nodes + from docutils.frontend import OptionParser + + from myst_parser.docutils_ import Parser as DocutilsParser + + stream = io.StringIO() + OptionParser( + components=(DocutilsParser,), + usage="myst-docutils-<writer> [options] [<source> [<destination>]]", + ).print_help(stream) + return [nodes.literal_block("", stream.getvalue())] + # app.connect("builder-inited", run_apidoc) app.add_css_file("custom.css") + app.add_directive("docutils-cli-help", DocutilsCliHelpDirective) diff --git a/docs/docutils.md b/docs/docutils.md new file mode 100644 index 00000000..48e36697 --- /dev/null +++ b/docs/docutils.md @@ -0,0 +1,79 @@ +(myst-docutils)= + +# MyST with Docutils + +Sphinx, and thus MyST-Parser, is built on top of the [Docutils](https://docutils.sourceforge.io/docs/) package. +MyST-Parser offers a renderer, parser and CLI-interface for working directly with Docutils, independent of Sphinx, as described below. + +:::{note} +Since these tools are independent of Sphinx, this means they cannot parse any Sphinx or Sphinx extensions specific roles or directives. +::: + +On installing MyST-Parser, the following CLI-commands are made available: + +- `myst-docutils-html`: converts MyST to HTML +- `myst-docutils-html5`: converts MyST to HTML5 +- `myst-docutils-latex`: converts MyST to LaTeX +- `myst-docutils-xml`: converts MyST to docutils-native XML +- `myst-docutils-pseudoxml`: converts MyST to pseudo-XML (to visualise the AST structure) + +Each command can be piped stdin or take a file path as an argument: + +```console +$ myst-docutils-html --help +$ echo "Hello World" | myst-docutils-html +$ myst-docutils-html hello-world.md +``` + +The commands are based on the [Docutils Front-End Tools](https://docutils.sourceforge.io/docs/user/tools.html), and so follow the same argument and options structure, included many of the MyST specific options detailed in [](sphinx/config-options). + +:::{dropdown} Shared Docutils CLI Options +```{docutils-cli-help} +``` +::: + +The CLI commands can also utilise the [`docutils.conf` configuration file](https://docutils.sourceforge.io/docs/user/config.html) to configure the behaviour of the CLI commands. For example: + +``` +# These entries affect all processing: +[general] +myst-enable-extensions: deflist,linkify +myst-footnote-transition: no + +# These entries affect specific HTML output: +[html writers] +embed-stylesheet: no + +[html5 writer] +stylesheet-dirs: path/to/html5_polyglot/ +stylesheet-path: minimal.css, responsive.css +``` + +You can also use the {py:class}`myst_parser.docutils_.Parser` class programmatically with the [Docutils publisher API](https://docutils.sourceforge.io/docs/api/publisher.html): + +```python +from docutils.core import publish_string +from myst_parser.docutils_ import Parser + +source = "hallo world\n: Definition" +output = publish_string( + source=source, + writer_name="html5", + settings_overrides={ + "myst_enable_extensions": ["deflist"], + "embed_stylesheet": False, + }, + parser=Parser(), +) +``` + +Finally, you can include MyST Markdown files within a RestructuredText file, using the [`include` directive](https://docutils.sourceforge.io/docs/ref/rst/directives.html#include): + +```rst +.. include:: include.md + :parser: myst_parser.docutils_ +``` + +```{important} +The `parser` option requires `docutils>=0.17` +``` diff --git a/docs/index.md b/docs/index.md index 08eeed08..93d0fa67 100644 --- a/docs/index.md +++ b/docs/index.md @@ -100,6 +100,7 @@ syntax/reference :caption: Topic Guides explain/index.md sphinx/index.md +docutils.md api/index.md develop/index.md ``` diff --git a/docs/sphinx/index.md b/docs/sphinx/index.md index 216e08af..decb14bb 100644 --- a/docs/sphinx/index.md +++ b/docs/sphinx/index.md @@ -1,3 +1,5 @@ +(myst-sphinx)= + # MyST with Sphinx The MyST Parser comes bundled with a Sphinx extension that allows you to use write Sphinx documentation entirely in MyST (or, in a combination of rST and MyST). diff --git a/docs/sphinx/use.md b/docs/sphinx/use.md index 3612940c..8b6df5df 100644 --- a/docs/sphinx/use.md +++ b/docs/sphinx/use.md @@ -32,6 +32,20 @@ To include rST, we must first "wrap" the directive in the [eval-rst directive](s .. include:: snippets/include-rst.rst ``` +(howto/include-md)= +## Include Markdown files into an rST file + +To include a MyST file within a ReStructuredText file, we can use the `parser` option of the `include` directive: + +```rst +.. include:: include.md + :parser: myst_parser.sphinx_ +``` + +```{important} +The `parser` option requires `docutils>=0.17` +``` + ## Use MyST in Jupyter Notebooks The [MyST-NB](https://myst-nb.readthedocs.io) tool provides a Sphinx extension for parsing **Jupyter Notebooks written with MyST Markdown**. It includes features like automatically executing notebooks during documentation builds, storing notebook cell outputs in order to insert them elsewhere in your documentation, and more. See the [MyST-NB documentation](https://myst-nb.readthedocs.io) for more information. diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index fd311c1b..81587bc5 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -40,7 +40,14 @@ def _validate( return _validate -DOCUTILS_UNSET = object() +class Unset: + """A sentinel class for unset settings.""" + + def __repr__(self): + return "UNSET" + + +DOCUTILS_UNSET = Unset() """Sentinel for arguments not set through docutils.conf.""" diff --git a/myst_parser/main.py b/myst_parser/main.py index 38153e6b..56df01a9 100644 --- a/myst_parser/main.py +++ b/myst_parser/main.py @@ -311,6 +311,10 @@ def to_docutils( def to_html(text: str, env=None, config: Optional[MdParserConfig] = None): + """Render text to HTML directly using markdown-it-py. + + This is mainly for test purposes only. + """ config = config or MdParserConfig() config.renderer = "html" md = default_parser(config) From f08e0e1c70557d8ca00bcabd5e8a6157f9522f7d Mon Sep 17 00:00:00 2001 From: Chris Sewell <chrisj_sewell@hotmail.com> Date: Sun, 5 Dec 2021 16:12:05 +0100 Subject: [PATCH 15/15] remove delattr --- myst_parser/docutils_.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/myst_parser/docutils_.py b/myst_parser/docutils_.py index 81587bc5..269a6ec2 100644 --- a/myst_parser/docutils_.py +++ b/myst_parser/docutils_.py @@ -3,7 +3,6 @@ .. include:: path/to/file.md :parser: myst_parser.docutils_ """ -from contextlib import suppress from typing import Any, Callable, Iterable, List, Optional, Tuple, Union from attr import Attribute @@ -138,8 +137,6 @@ def create_myst_config(settings: frontend.Values): continue setting = f"myst_{attribute.name}" val = getattr(settings, setting, DOCUTILS_UNSET) - with suppress(AttributeError): - delattr(settings, setting) if val is not DOCUTILS_UNSET: values[attribute.name] = val values["renderer"] = "docutils"