Remove whitespace tokenizing rule in markdown lexer (#1008)

fix #744 related issue: go-gitea/gitea#32136 current markdown lexer is tokenizing by whitespace, but in markdown sytanx this implementation can cause some errors.
alecthomas · Nov 14, 2024 · 539d031 · 539d031
1 parent 6622bd5
commit 539d031
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 2 deletions.
diff --git a/lexers/markdown.go b/lexers/markdown.go
@@ -40,7 +40,6 @@ func markdownRules() Rules {
 			{"`[^`]+`", LiteralStringBacktick, nil},
 			{`[@#][\w/:]+`, NameEntity, nil},
 			{`(!?\[)([^]]+)(\])(\()([^)]+)(\))`, ByGroups(Text, NameTag, Text, Text, NameAttribute, Text), nil},
-			{`[^\\\s]+`, Other, nil},
 			{`.|\n`, Other, nil},
 		},
 	}

diff --git a/lexers/testdata/markdown.actual b/lexers/testdata/markdown.actual
@@ -85,3 +85,9 @@ end loop;
 end
 $$;
 ```
+
+## MarkdownLink Test
+[normal link](https://google.com)  
+  [whitespace before link](https://google.com)  
+abc[non-whitespace before link](https://google.com)  
+([first link](https://google.com) and [second link](https://google.com))  
diff --git a/lexers/testdata/markdown.expected b/lexers/testdata/markdown.expected
@@ -441,5 +441,27 @@
   {"type":"Punctuation","value":";"},
   {"type":"Text","value":"\n"},
   {"type":"LiteralString","value":"```"},
-  {"type":"Text","value":"\n"}
+  {"type":"Text","value":"\n\n"},
+  {"type":"GenericSubheading","value":"## MarkdownLink Test\n"},
+  {"type":"Text","value":"["},
+  {"type":"NameTag","value":"normal link"},
+  {"type":"Text","value":"]("},
+  {"type":"NameAttribute","value":"https://google.com"},
+  {"type":"Text","value":")  \n  ["},
+  {"type":"NameTag","value":"whitespace before link"},
+  {"type":"Text","value":"]("},
+  {"type":"NameAttribute","value":"https://google.com"},
+  {"type":"Text","value":")  \nabc["},
+  {"type":"NameTag","value":"non-whitespace before link"},
+  {"type":"Text","value":"]("},
+  {"type":"NameAttribute","value":"https://google.com"},
+  {"type":"Text","value":")  \n(["},
+  {"type":"NameTag","value":"first link"},
+  {"type":"Text","value":"]("},
+  {"type":"NameAttribute","value":"https://google.com"},
+  {"type":"Text","value":") and ["},
+  {"type":"NameTag","value":"second link"},
+  {"type":"Text","value":"]("},
+  {"type":"NameAttribute","value":"https://google.com"},
+  {"type":"Text","value":"))  \n"}
 ]