From 03cf85b024244137776e05129d9072127cf4a2b6 Mon Sep 17 00:00:00 2001 From: Michael Schmidt Date: Sat, 18 May 2019 20:56:47 +0200 Subject: [PATCH 1/5] Added support for nested bold and italic tokens --- components/prism-markdown.js | 405 +++++++++--------- components/prism-markdown.min.js | 2 +- tests/languages/markdown/bold_feature.test | 73 ++-- tests/languages/markdown/italic_feature.test | 27 +- .../markdown/nested_bold_italic_feature.test | 162 +++++++ tests/languages/markdown/strike_feature.test | 57 ++- 6 files changed, 479 insertions(+), 247 deletions(-) create mode 100644 tests/languages/markdown/nested_bold_italic_feature.test diff --git a/components/prism-markdown.js b/components/prism-markdown.js index 18a5154cb1..bd5008a8d9 100644 --- a/components/prism-markdown.js +++ b/components/prism-markdown.js @@ -1,227 +1,250 @@ -Prism.languages.markdown = Prism.languages.extend('markup', {}); -Prism.languages.insertBefore('markdown', 'prolog', { - 'blockquote': { - // > ... - pattern: /^>(?:[\t ]*>)*/m, - alias: 'punctuation' - }, - 'code': [ - { - // Prefixed by 4 spaces or 1 tab - pattern: /^(?: {4}|\t).+/m, - alias: 'keyword' +(function (Prism) { + + // Allow only one line break + var inner = /\\.|[^\\\n\r_]|(?:\r?\n|\r)(?!\r?\n|\r)/.source; + + // both bold and italic allow one nested instance of the other + var bold_ = /__(?:|_(?:)+_)+__/.source.replace(//g, inner); + var italic_ = /_(?:|__(?:)+__)+_/.source.replace(//g, inner); + + Prism.languages.markdown = Prism.languages.extend('markup', {}); + Prism.languages.insertBefore('markdown', 'prolog', { + 'blockquote': { + // > ... + pattern: /^>(?:[\t ]*>)*/m, + alias: 'punctuation' }, - { - // `code` - // ``code`` - pattern: /``.+?``|`[^`\n]+`/, - alias: 'keyword' + 'code': [ + { + // Prefixed by 4 spaces or 1 tab + pattern: /^(?: {4}|\t).+/m, + alias: 'keyword' + }, + { + // `code` + // ``code`` + pattern: /``.+?``|`[^`\n]+`/, + alias: 'keyword' + }, + { + // ```optional language + // code block + // ``` + pattern: /^```[\s\S]*?^```$/m, + greedy: true, + inside: { + 'code-block': { + pattern: /^(```.*(?:\r?\n|\r))[\s\S]+?(?=(?:\r?\n|\r)^```$)/m, + lookbehind: true + }, + 'code-language': { + pattern: /^(```).+/, + lookbehind: true + }, + 'punctuation': /```/ + } + } + ], + 'title': [ + { + // title 1 + // ======= + + // title 2 + // ------- + pattern: /\S.*(?:\r?\n|\r)(?:==+|--+)/, + alias: 'important', + inside: { + punctuation: /==+$|--+$/ + } + }, + { + // # title 1 + // ###### title 6 + pattern: /(^\s*)#+.+/m, + lookbehind: true, + alias: 'important', + inside: { + punctuation: /^#+|#+$/ + } + } + ], + 'hr': { + // *** + // --- + // * * * + // ----------- + pattern: /(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m, + lookbehind: true, + alias: 'punctuation' }, - { - // ```optional language - // code block - // ``` - pattern: /^```[\s\S]*?^```$/m, - greedy: true, + 'list': { + // * item + // + item + // - item + // 1. item + pattern: /(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m, + lookbehind: true, + alias: 'punctuation' + }, + 'url-reference': { + // [id]: http://example.com "Optional title" + // [id]: http://example.com 'Optional title' + // [id]: http://example.com (Optional title) + // [id]: "Optional title" + pattern: /!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/, inside: { - 'code-block': { - pattern: /^(```.*(?:\r?\n|\r))[\s\S]+?(?=(?:\r?\n|\r)^```$)/m, + 'variable': { + pattern: /^(!?\[)[^\]]+/, lookbehind: true }, - 'code-language': { - pattern: /^(```).+/, - lookbehind: true + 'string': /(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/, + 'punctuation': /^[\[\]!:]|[<>]/ + }, + alias: 'url' + }, + 'bold': { + // **strong** + // __strong__ + + pattern: RegExp('((?:^|[^\\\\])(?:\\\\{2})*)(?:' + bold_ + '|' + bold_.replace(/_/g, '\\*') + ')'), + lookbehind: true, + greedy: true, + inside: { + 'content': { + pattern: /(^..)[\s\S]+(?=..$)/, + lookbehind: true, + inside: {} // see below }, - 'punctuation': /```/ + 'punctuation': /\*\*|__/ } - } - ], - 'title': [ - { - // title 1 - // ======= - - // title 2 - // ------- - pattern: /\S.*(?:\r?\n|\r)(?:==+|--+)/, - alias: 'important', + }, + 'italic': { + // *em* + // _em_ + + pattern: RegExp('((?:^|[^\\\\])(?:\\\\{2})*)(?:' + italic_ + '|' + italic_.replace(/_/g, '\\*') + ')'), + lookbehind: true, + greedy: true, inside: { - punctuation: /==+$|--+$/ + 'content': { + pattern: /(^.)[\s\S]+(?=.$)/, + lookbehind: true, + inside: {} // see below + }, + 'punctuation': /[*_]/ } }, - { - // # title 1 - // ###### title 6 - pattern: /(^\s*)#+.+/m, + 'strike': { + // ~~strike through~~ + // ~strike~ + + pattern: RegExp(/((?:^|[^\\])(?:\\{2})*)(~~?)(?:|_)+?\2/.source.replace(//g, inner)), lookbehind: true, - alias: 'important', + greedy: true, inside: { - punctuation: /^#+|#+$/ + 'content': { + pattern: /(^~~?)[\s\S]+(?=\1$)/, + lookbehind: true, + inside: {} // see below + }, + 'punctuation': /~~?/ } - } - ], - 'hr': { - // *** - // --- - // * * * - // ----------- - pattern: /(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m, - lookbehind: true, - alias: 'punctuation' - }, - 'list': { - // * item - // + item - // - item - // 1. item - pattern: /(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m, - lookbehind: true, - alias: 'punctuation' - }, - 'url-reference': { - // [id]: http://example.com "Optional title" - // [id]: http://example.com 'Optional title' - // [id]: http://example.com (Optional title) - // [id]: "Optional title" - pattern: /!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/, - inside: { - 'variable': { - pattern: /^(!?\[)[^\]]+/, - lookbehind: true - }, - 'string': /(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/, - 'punctuation': /^[\[\]!:]|[<>]/ }, - alias: 'url' - }, - 'bold': { - // **strong** - // __strong__ - - // Allow only one line break - pattern: /(^|[^\\])(\*\*|__)(?:(?:\r?\n|\r)(?!\r?\n|\r)|.)+?\2/, - lookbehind: true, - greedy: true, - inside: { - 'punctuation': /^\*\*|^__|\*\*$|__$/ - } - }, - 'italic': { - // *em* - // _em_ - - // Allow only one line break - pattern: /(^|[^\\])([*_])(?:(?:\r?\n|\r)(?!\r?\n|\r)|.)+?\2/, - lookbehind: true, - greedy: true, - inside: { - 'punctuation': /^[*_]|[*_]$/ - } - }, - 'strike': { - // ~~strike through~~ - // ~strike~ - - // Allow only one line break - pattern: /(^|[^\\])(~~?)(?:(?:\r?\n|\r)(?!\r?\n|\r)|.)+?\2/, - lookbehind: true, - greedy: true, - inside: { - 'punctuation': /^~~?|~~?$/ - } - }, - 'url': { - // [example](http://example.com "Optional title") - // [example] [id] - pattern: /!?\[[^\]]+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[[^\]\n]*\])/, - inside: { - 'variable': { - pattern: /(!?\[)[^\]]+(?=\]$)/, - lookbehind: true - }, - 'string': { - pattern: /"(?:\\.|[^"\\])*"(?=\)$)/ + 'url': { + // [example](http://example.com "Optional title") + // [example] [id] + pattern: /!?\[[^\]]+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[[^\]\n]*\])/, + inside: { + 'variable': { + pattern: /(!?\[)[^\]]+(?=\]$)/, + lookbehind: true + }, + 'string': { + pattern: /"(?:\\.|[^"\\])*"(?=\)$)/ + } } } - } -}); - -['bold', 'italic', 'strike'].forEach(function (token) { - ['url', 'bold', 'italic', 'strike'].forEach(function (inside) { - if (token !== inside) { - Prism.languages.markdown[token].inside[inside] = Prism.languages.markdown[inside]; - } }); -}); -Prism.hooks.add('after-tokenize', function (env) { - if (env.language !== 'markdown' && env.language !== 'md') { - return; - } + ['bold', 'italic', 'strike'].forEach(function (token) { + ['url', 'bold', 'italic', 'strike'].forEach(function (inside) { + if (token !== inside) { + Prism.languages.markdown[token].inside.content.inside[inside] = Prism.languages.markdown[inside]; + } + }); + }); - function walkTokens(tokens) { - if (!tokens || typeof tokens === 'string') { + Prism.hooks.add('after-tokenize', function (env) { + if (env.language !== 'markdown' && env.language !== 'md') { return; } - for (var i = 0, l = tokens.length; i < l; i++) { - var token = tokens[i]; - - if (token.type !== 'code') { - walkTokens(token.content); - continue; + function walkTokens(tokens) { + if (!tokens || typeof tokens === 'string') { + return; } - var codeLang = token.content[1]; - var codeBlock = token.content[3]; + for (var i = 0, l = tokens.length; i < l; i++) { + var token = tokens[i]; + + if (token.type !== 'code') { + walkTokens(token.content); + continue; + } + + var codeLang = token.content[1]; + var codeBlock = token.content[3]; - if (codeLang && codeBlock && - codeLang.type === 'code-language' && codeBlock.type === 'code-block' && - typeof codeLang.content === 'string') { + if (codeLang && codeBlock && + codeLang.type === 'code-language' && codeBlock.type === 'code-block' && + typeof codeLang.content === 'string') { - // this might be a language that Prism does not support - var alias = 'language-' + codeLang.content.trim().split(/\s+/)[0].toLowerCase(); + // this might be a language that Prism does not support + var alias = 'language-' + codeLang.content.trim().split(/\s+/)[0].toLowerCase(); - // add alias - if (!codeBlock.alias) { - codeBlock.alias = [alias]; - } else if (typeof codeBlock.alias === 'string') { - codeBlock.alias = [codeBlock.alias, alias]; - } else { - codeBlock.alias.push(alias); + // add alias + if (!codeBlock.alias) { + codeBlock.alias = [alias]; + } else if (typeof codeBlock.alias === 'string') { + codeBlock.alias = [codeBlock.alias, alias]; + } else { + codeBlock.alias.push(alias); + } } } } - } - - walkTokens(env.tokens); -}); - -Prism.hooks.add('wrap', function (env) { - if (env.type !== 'code-block') { - return; - } - - var codeLang = ''; - for (var i = 0, l = env.classes.length; i < l; i++) { - var cls = env.classes[i]; - var match = /language-(.+)/.exec(cls); - if (match) { - codeLang = match[1]; - break; + + walkTokens(env.tokens); + }); + + Prism.hooks.add('wrap', function (env) { + if (env.type !== 'code-block') { + return; } - } - var grammar = Prism.languages[codeLang]; + var codeLang = ''; + for (var i = 0, l = env.classes.length; i < l; i++) { + var cls = env.classes[i]; + var match = /language-(.+)/.exec(cls); + if (match) { + codeLang = match[1]; + break; + } + } - if (!grammar) { - return; - } + var grammar = Prism.languages[codeLang]; - // reverse Prism.util.encode - var code = env.content.replace(/</g, '<').replace(/&/g, '&'); + if (!grammar) { + return; + } + + // reverse Prism.util.encode + var code = env.content.replace(/</g, '<').replace(/&/g, '&'); + + env.content = Prism.highlight(code, grammar, codeLang); + }); - env.content = Prism.highlight(code, grammar, codeLang); -}); + Prism.languages.md = Prism.languages.markdown; -Prism.languages.md = Prism.languages.markdown; +}(Prism)); diff --git a/components/prism-markdown.min.js b/components/prism-markdown.min.js index b42f979300..95596fd6d8 100644 --- a/components/prism-markdown.min.js +++ b/components/prism-markdown.min.js @@ -1 +1 @@ -Prism.languages.markdown=Prism.languages.extend("markup",{}),Prism.languages.insertBefore("markdown","prolog",{blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},code:[{pattern:/^(?: {4}|\t).+/m,alias:"keyword"},{pattern:/``.+?``|`[^`\n]+`/,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\r?\n|\r))[\s\S]+?(?=(?:\r?\n|\r)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\r?\n|\r)(?:==+|--+)/,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#+.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:/(^|[^\\])(\*\*|__)(?:(?:\r?\n|\r)(?!\r?\n|\r)|.)+?\2/,lookbehind:!0,greedy:!0,inside:{punctuation:/^\*\*|^__|\*\*$|__$/}},italic:{pattern:/(^|[^\\])([*_])(?:(?:\r?\n|\r)(?!\r?\n|\r)|.)+?\2/,lookbehind:!0,greedy:!0,inside:{punctuation:/^[*_]|[*_]$/}},strike:{pattern:/(^|[^\\])(~~?)(?:(?:\r?\n|\r)(?!\r?\n|\r)|.)+?\2/,lookbehind:!0,greedy:!0,inside:{punctuation:/^~~?|~~?$/}},url:{pattern:/!?\[[^\]]+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[[^\]\n]*\])/,inside:{variable:{pattern:/(!?\[)[^\]]+(?=\]$)/,lookbehind:!0},string:{pattern:/"(?:\\.|[^"\\])*"(?=\)$)/}}}}),["bold","italic","strike"].forEach(function(a){["url","bold","italic","strike"].forEach(function(n){a!==n&&(Prism.languages.markdown[a].inside[n]=Prism.languages.markdown[n])})}),Prism.hooks.add("after-tokenize",function(n){"markdown"!==n.language&&"md"!==n.language||!function n(a){if(a&&"string"!=typeof a)for(var t=0,e=a.length;t/g,n),t="_(?:|__(?:)+__)+_".replace(//g,n);s.languages.markdown=s.languages.extend("markup",{}),s.languages.insertBefore("markdown","prolog",{blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},code:[{pattern:/^(?: {4}|\t).+/m,alias:"keyword"},{pattern:/``.+?``|`[^`\n]+`/,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\r?\n|\r))[\s\S]+?(?=(?:\r?\n|\r)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\r?\n|\r)(?:==+|--+)/,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#+.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(?:"+e+"|"+e.replace(/_/g,"\\*")+")"),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(?:"+t+"|"+t.replace(/_/g,"\\*")+")"),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(~~?)(?:|_)+?\\2".replace(//g,n)),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},url:{pattern:/!?\[[^\]]+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[[^\]\n]*\])/,inside:{variable:{pattern:/(!?\[)[^\]]+(?=\]$)/,lookbehind:!0},string:{pattern:/"(?:\\.|[^"\\])*"(?=\)$)/}}}}),["bold","italic","strike"].forEach(function(e){["url","bold","italic","strike"].forEach(function(n){e!==n&&(s.languages.markdown[e].inside.content.inside[n]=s.languages.markdown[n])})}),s.hooks.add("after-tokenize",function(n){"markdown"!==n.language&&"md"!==n.language||!function n(e){if(e&&"string"!=typeof e)for(var t=0,a=e.length;t Date: Mon, 24 Jun 2019 14:47:41 +0200 Subject: [PATCH 2/5] Improved tests --- tests/languages/markdown/bold_feature.test | 92 ++++++++++++ tests/languages/markdown/italic_feature.test | 140 +++++++++++++++++++ 2 files changed, 232 insertions(+) diff --git a/tests/languages/markdown/bold_feature.test b/tests/languages/markdown/bold_feature.test index 26dc52687f..db915b9131 100644 --- a/tests/languages/markdown/bold_feature.test +++ b/tests/languages/markdown/bold_feature.test @@ -8,8 +8,15 @@ bar__ __foo*bar*baz__ __foo_bar_baz__ __foo~bar~baz__ +__foo~~bar~~baz__ __foo[bar](baz)__ +**foo*bar*baz** +**foo_bar_baz** +**foo~bar~baz** +**foo~~bar~~baz** +**foo[bar](baz)** + ---------------------------------------------------- [ @@ -86,6 +93,21 @@ __foo[bar](baz)__ ]], ["punctuation", "__"] ]], + ["bold", [ + ["punctuation", "__"], + ["content", [ + "foo", + ["strike", [ + ["punctuation", "~~"], + ["content", [ + "bar" + ]], + ["punctuation", "~~"] + ]], + "baz" + ]], + ["punctuation", "__"] + ]], ["bold", [ ["punctuation", "__"], ["content", [ @@ -95,6 +117,76 @@ __foo[bar](baz)__ ]] ]], ["punctuation", "__"] + ]], + ["bold", [ + ["punctuation", "**"], + ["content", [ + "foo", + ["italic", [ + ["punctuation", "*"], + ["content", [ + "bar" + ]], + ["punctuation", "*"] + ]], + "baz" + ]], + ["punctuation", "**"] + ]], + ["bold", [ + ["punctuation", "**"], + ["content", [ + "foo", + ["italic", [ + ["punctuation", "_"], + ["content", [ + "bar" + ]], + ["punctuation", "_"] + ]], + "baz" + ]], + ["punctuation", "**"] + ]], + ["bold", [ + ["punctuation", "**"], + ["content", [ + "foo", + ["strike", [ + ["punctuation", "~"], + ["content", [ + "bar" + ]], + ["punctuation", "~"] + ]], + "baz" + ]], + ["punctuation", "**"] + ]], + ["bold", [ + ["punctuation", "**"], + ["content", [ + "foo", + ["strike", [ + ["punctuation", "~~"], + ["content", [ + "bar" + ]], + ["punctuation", "~~"] + ]], + "baz" + ]], + ["punctuation", "**"] + ]], + ["bold", [ + ["punctuation", "**"], + ["content", [ + "foo", + ["url", [ + "[bar](baz)" + ]] + ]], + ["punctuation", "**"] ]] ] diff --git a/tests/languages/markdown/italic_feature.test b/tests/languages/markdown/italic_feature.test index 2003864227..fc9c6c858e 100644 --- a/tests/languages/markdown/italic_feature.test +++ b/tests/languages/markdown/italic_feature.test @@ -5,8 +5,18 @@ _foobar_ _foo bar_ +_foo__bar__baz_ +_foo**bar**baz_ +_foo~bar~baz_ +_foo~~bar~~baz_ _foo[bar](baz)_ +*foo__bar__baz* +*foo**bar**baz* +*foo~bar~baz* +*foo~~bar~~baz* +*foo[bar](baz)* + ---------------------------------------------------- [ @@ -38,6 +48,66 @@ _foo[bar](baz)_ ]], ["punctuation", "_"] ]], + ["italic", [ + ["punctuation", "_"], + ["content", [ + "foo", + ["bold", [ + ["punctuation", "__"], + ["content", [ + "bar" + ]], + ["punctuation", "__"] + ]], + "baz" + ]], + ["punctuation", "_"] + ]], + ["italic", [ + ["punctuation", "_"], + ["content", [ + "foo", + ["bold", [ + ["punctuation", "**"], + ["content", [ + "bar" + ]], + ["punctuation", "**"] + ]], + "baz" + ]], + ["punctuation", "_"] + ]], + ["italic", [ + ["punctuation", "_"], + ["content", [ + "foo", + ["strike", [ + ["punctuation", "~"], + ["content", [ + "bar" + ]], + ["punctuation", "~"] + ]], + "baz" + ]], + ["punctuation", "_"] + ]], + ["italic", [ + ["punctuation", "_"], + ["content", [ + "foo", + ["strike", [ + ["punctuation", "~~"], + ["content", [ + "bar" + ]], + ["punctuation", "~~"] + ]], + "baz" + ]], + ["punctuation", "_"] + ]], ["italic", [ ["punctuation", "_"], ["content", [ @@ -47,6 +117,76 @@ _foo[bar](baz)_ ]] ]], ["punctuation", "_"] + ]], + ["italic", [ + ["punctuation", "*"], + ["content", [ + "foo", + ["bold", [ + ["punctuation", "__"], + ["content", [ + "bar" + ]], + ["punctuation", "__"] + ]], + "baz" + ]], + ["punctuation", "*"] + ]], + ["italic", [ + ["punctuation", "*"], + ["content", [ + "foo", + ["bold", [ + ["punctuation", "**"], + ["content", [ + "bar" + ]], + ["punctuation", "**"] + ]], + "baz" + ]], + ["punctuation", "*"] + ]], + ["italic", [ + ["punctuation", "*"], + ["content", [ + "foo", + ["strike", [ + ["punctuation", "~"], + ["content", [ + "bar" + ]], + ["punctuation", "~"] + ]], + "baz" + ]], + ["punctuation", "*"] + ]], + ["italic", [ + ["punctuation", "*"], + ["content", [ + "foo", + ["strike", [ + ["punctuation", "~~"], + ["content", [ + "bar" + ]], + ["punctuation", "~~"] + ]], + "baz" + ]], + ["punctuation", "*"] + ]], + ["italic", [ + ["punctuation", "*"], + ["content", [ + "foo", + ["url", [ + "[bar](baz)" + ]] + ]], + ["punctuation", "*"] ]] ] From 986c9de0b55fcf71cc82606d9d5bcb8e59c0c079 Mon Sep 17 00:00:00 2001 From: RunDevelopment Date: Mon, 24 Jun 2019 15:43:49 +0200 Subject: [PATCH 3/5] More strike tests --- tests/languages/markdown/strike_feature.test | 108 +++++++++++++++++++ 1 file changed, 108 insertions(+) diff --git a/tests/languages/markdown/strike_feature.test b/tests/languages/markdown/strike_feature.test index 11809351dd..414fb8bdc0 100644 --- a/tests/languages/markdown/strike_feature.test +++ b/tests/languages/markdown/strike_feature.test @@ -6,9 +6,17 @@ bar~~ bar~ ~foo*bar*baz~ +~foo_bar_baz~ +~foo**bar**baz~ ~foo__bar__baz~ ~foo[bar](baz)~ +~~foo*bar*baz~~ +~~foo_bar_baz~~ +~~foo**bar**baz~~ +~~foo__bar__baz~~ +~~foo[bar](baz)~~ + ---------------------------------------------------- [ @@ -55,6 +63,36 @@ bar~ ]], ["punctuation", "~"] ]], + ["strike", [ + ["punctuation", "~"], + ["content", [ + "foo", + ["italic", [ + ["punctuation", "_"], + ["content", [ + "bar" + ]], + ["punctuation", "_"] + ]], + "baz" + ]], + ["punctuation", "~"] + ]], + ["strike", [ + ["punctuation", "~"], + ["content", [ + "foo", + ["bold", [ + ["punctuation", "**"], + ["content", [ + "bar" + ]], + ["punctuation", "**"] + ]], + "baz" + ]], + ["punctuation", "~"] + ]], ["strike", [ ["punctuation", "~"], ["content", [ @@ -79,6 +117,76 @@ bar~ ]] ]], ["punctuation", "~"] + ]], + ["strike", [ + ["punctuation", "~~"], + ["content", [ + "foo", + ["italic", [ + ["punctuation", "*"], + ["content", [ + "bar" + ]], + ["punctuation", "*"] + ]], + "baz" + ]], + ["punctuation", "~~"] + ]], + ["strike", [ + ["punctuation", "~~"], + ["content", [ + "foo", + ["italic", [ + ["punctuation", "_"], + ["content", [ + "bar" + ]], + ["punctuation", "_"] + ]], + "baz" + ]], + ["punctuation", "~~"] + ]], + ["strike", [ + ["punctuation", "~~"], + ["content", [ + "foo", + ["bold", [ + ["punctuation", "**"], + ["content", [ + "bar" + ]], + ["punctuation", "**"] + ]], + "baz" + ]], + ["punctuation", "~~"] + ]], + ["strike", [ + ["punctuation", "~~"], + ["content", [ + "foo", + ["bold", [ + ["punctuation", "__"], + ["content", [ + "bar" + ]], + ["punctuation", "__"] + ]], + "baz" + ]], + ["punctuation", "~~"] + ]], + ["strike", [ + ["punctuation", "~~"], + ["content", [ + "foo", + ["url", [ + "[bar](baz)" + ]] + ]], + ["punctuation", "~~"] ]] ] From 9c75843a59b496b9a7dc5b154473cbeac2b02a22 Mon Sep 17 00:00:00 2001 From: RunDevelopment Date: Mon, 24 Jun 2019 15:48:11 +0200 Subject: [PATCH 4/5] Improved inline pattern creation + more comments --- components/prism-markdown.js | 45 +++++++++++++++++++++++++++----- components/prism-markdown.min.js | 2 +- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/components/prism-markdown.js b/components/prism-markdown.js index bd5008a8d9..94adc834b1 100644 --- a/components/prism-markdown.js +++ b/components/prism-markdown.js @@ -3,9 +3,25 @@ // Allow only one line break var inner = /\\.|[^\\\n\r_]|(?:\r?\n|\r)(?!\r?\n|\r)/.source; - // both bold and italic allow one nested instance of the other - var bold_ = /__(?:|_(?:)+_)+__/.source.replace(//g, inner); - var italic_ = /_(?:|__(?:)+__)+_/.source.replace(//g, inner); + /** + * This function is intended for the creation of the bold or italic pattern. + * + * This also adds a lookbehind group to the given pattern to ensure that the pattern is not backslash-escaped. + * + * _Note:_ Keep in mind that this adds a capturing group. + * + * @param {string} pattern + * @param {boolean} starAlternative Whether to also add an alternative where all `_`s are replaced with `*`s. + * @returns {RegExp} + */ + function createInline(pattern, starAlternative) { + pattern = pattern.replace(//g, inner); + if (starAlternative) { + pattern = pattern + '|' + pattern.replace(/_/g, '\\*'); + } + return RegExp(/((?:^|[^\\])(?:\\{2})*)/.source + '(?:' + pattern + ')'); + } + Prism.languages.markdown = Prism.languages.extend('markup', {}); Prism.languages.insertBefore('markdown', 'prolog', { @@ -107,7 +123,8 @@ // **strong** // __strong__ - pattern: RegExp('((?:^|[^\\\\])(?:\\\\{2})*)(?:' + bold_ + '|' + bold_.replace(/_/g, '\\*') + ')'), + // allow one nested instance of italic text using the same delimiter + pattern: createInline(/__(?:|_(?:)+_)+__/.source, true), lookbehind: true, greedy: true, inside: { @@ -123,7 +140,8 @@ // *em* // _em_ - pattern: RegExp('((?:^|[^\\\\])(?:\\\\{2})*)(?:' + italic_ + '|' + italic_.replace(/_/g, '\\*') + ')'), + // allow one nested instance of bold text using the same delimiter + pattern: createInline(/_(?:|__(?:)+__)+_/.source, true), lookbehind: true, greedy: true, inside: { @@ -139,7 +157,8 @@ // ~~strike through~~ // ~strike~ - pattern: RegExp(/((?:^|[^\\])(?:\\{2})*)(~~?)(?:|_)+?\2/.source.replace(//g, inner)), + // extra _ is because the inner pattern intentionally doesn't include it because of bold and italic + pattern: createInline(/(~~?)(?:|_)+?\2/.source, false), lookbehind: true, greedy: true, inside: { @@ -193,6 +212,20 @@ continue; } + /* + * Add the correct `language-xxxx` class to this code block. Keep in mind that the `code-language` token + * is optional. But the grammar is defined so that there is only one case we have to handle: + * + * token.content = [ + * ```, + * xxxx, + * '\n', // exactly one new lines (\r or \n or \r\n) + * ..., + * '\n', // exactly one new lines again + * ``` + * ]; + */ + var codeLang = token.content[1]; var codeBlock = token.content[3]; diff --git a/components/prism-markdown.min.js b/components/prism-markdown.min.js index 95596fd6d8..22eba7ada9 100644 --- a/components/prism-markdown.min.js +++ b/components/prism-markdown.min.js @@ -1 +1 @@ -!function(s){var n="\\\\.|[^\\\\\\n\r_]|(?:\r?\n|\r)(?!\r?\n|\r)",e="__(?:|_(?:)+_)+__".replace(//g,n),t="_(?:|__(?:)+__)+_".replace(//g,n);s.languages.markdown=s.languages.extend("markup",{}),s.languages.insertBefore("markdown","prolog",{blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},code:[{pattern:/^(?: {4}|\t).+/m,alias:"keyword"},{pattern:/``.+?``|`[^`\n]+`/,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\r?\n|\r))[\s\S]+?(?=(?:\r?\n|\r)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\r?\n|\r)(?:==+|--+)/,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#+.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(?:"+e+"|"+e.replace(/_/g,"\\*")+")"),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(?:"+t+"|"+t.replace(/_/g,"\\*")+")"),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(~~?)(?:|_)+?\\2".replace(//g,n)),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},url:{pattern:/!?\[[^\]]+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[[^\]\n]*\])/,inside:{variable:{pattern:/(!?\[)[^\]]+(?=\]$)/,lookbehind:!0},string:{pattern:/"(?:\\.|[^"\\])*"(?=\)$)/}}}}),["bold","italic","strike"].forEach(function(e){["url","bold","italic","strike"].forEach(function(n){e!==n&&(s.languages.markdown[e].inside.content.inside[n]=s.languages.markdown[n])})}),s.hooks.add("after-tokenize",function(n){"markdown"!==n.language&&"md"!==n.language||!function n(e){if(e&&"string"!=typeof e)for(var t=0,a=e.length;t/g,"\\\\.|[^\\\\\\n\r_]|(?:\r?\n|\r)(?!\r?\n|\r)"),e&&(n=n+"|"+n.replace(/_/g,"\\*")),RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(?:"+n+")")}s.languages.markdown=s.languages.extend("markup",{}),s.languages.insertBefore("markdown","prolog",{blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},code:[{pattern:/^(?: {4}|\t).+/m,alias:"keyword"},{pattern:/``.+?``|`[^`\n]+`/,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\r?\n|\r))[\s\S]+?(?=(?:\r?\n|\r)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\r?\n|\r)(?:==+|--+)/,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#+.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:n("__(?:|_(?:)+_)+__",!0),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:n("_(?:|__(?:)+__)+_",!0),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:n("(~~?)(?:|_)+?\\2"),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},url:{pattern:/!?\[[^\]]+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[[^\]\n]*\])/,inside:{variable:{pattern:/(!?\[)[^\]]+(?=\]$)/,lookbehind:!0},string:{pattern:/"(?:\\.|[^"\\])*"(?=\)$)/}}}}),["bold","italic","strike"].forEach(function(e){["url","bold","italic","strike"].forEach(function(n){e!==n&&(s.languages.markdown[e].inside.content.inside[n]=s.languages.markdown[n])})}),s.hooks.add("after-tokenize",function(n){"markdown"!==n.language&&"md"!==n.language||!function n(e){if(e&&"string"!=typeof e)for(var t=0,a=e.length;t Date: Mon, 24 Jun 2019 15:51:46 +0200 Subject: [PATCH 5/5] Rebuilt Prism --- components/prism-markdown.min.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/prism-markdown.min.js b/components/prism-markdown.min.js index 22eba7ada9..fd5615fdf5 100644 --- a/components/prism-markdown.min.js +++ b/components/prism-markdown.min.js @@ -1 +1 @@ -!function(s){function n(n,e){return n=n.replace(//g,"\\\\.|[^\\\\\\n\r_]|(?:\r?\n|\r)(?!\r?\n|\r)"),e&&(n=n+"|"+n.replace(/_/g,"\\*")),RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(?:"+n+")")}s.languages.markdown=s.languages.extend("markup",{}),s.languages.insertBefore("markdown","prolog",{blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},code:[{pattern:/^(?: {4}|\t).+/m,alias:"keyword"},{pattern:/``.+?``|`[^`\n]+`/,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\r?\n|\r))[\s\S]+?(?=(?:\r?\n|\r)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\r?\n|\r)(?:==+|--+)/,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#+.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:n("__(?:|_(?:)+_)+__",!0),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:n("_(?:|__(?:)+__)+_",!0),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:n("(~~?)(?:|_)+?\\2"),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},url:{pattern:/!?\[[^\]]+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[[^\]\n]*\])/,inside:{variable:{pattern:/(!?\[)[^\]]+(?=\]$)/,lookbehind:!0},string:{pattern:/"(?:\\.|[^"\\])*"(?=\)$)/}}}}),["bold","italic","strike"].forEach(function(e){["url","bold","italic","strike"].forEach(function(n){e!==n&&(s.languages.markdown[e].inside.content.inside[n]=s.languages.markdown[n])})}),s.hooks.add("after-tokenize",function(n){"markdown"!==n.language&&"md"!==n.language||!function n(e){if(e&&"string"!=typeof e)for(var t=0,a=e.length;t/g,"\\\\.|[^\\\\\\n\r_]|(?:\r?\n|\r)(?!\r?\n|\r)"),e&&(n=n+"|"+n.replace(/_/g,"\\*")),RegExp("((?:^|[^\\\\])(?:\\\\{2})*)(?:"+n+")")}s.languages.markdown=s.languages.extend("markup",{}),s.languages.insertBefore("markdown","prolog",{blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},code:[{pattern:/^(?: {4}|\t).+/m,alias:"keyword"},{pattern:/``.+?``|`[^`\n]+`/,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\r?\n|\r))[\s\S]+?(?=(?:\r?\n|\r)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\r?\n|\r)(?:==+|--+)/,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#+.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:n("__(?:|_(?:)+_)+__",!0),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:n("_(?:|__(?:)+__)+_",!0),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:n("(~~?)(?:|_)+?\\2",!1),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},url:{pattern:/!?\[[^\]]+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[[^\]\n]*\])/,inside:{variable:{pattern:/(!?\[)[^\]]+(?=\]$)/,lookbehind:!0},string:{pattern:/"(?:\\.|[^"\\])*"(?=\)$)/}}}}),["bold","italic","strike"].forEach(function(e){["url","bold","italic","strike"].forEach(function(n){e!==n&&(s.languages.markdown[e].inside.content.inside[n]=s.languages.markdown[n])})}),s.hooks.add("after-tokenize",function(n){"markdown"!==n.language&&"md"!==n.language||!function n(e){if(e&&"string"!=typeof e)for(var t=0,a=e.length;t