feat: multiple overwrite sections using markdig (#8457)

dotnet · Mar 7, 2023 · f3bdb0b · f3bdb0b
1 parent 2ef466a
commit f3bdb0b
Show file tree

Hide file tree

Showing 26 changed files with 218 additions and 248 deletions.
diff --git a/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildParameters.cs b/src/Microsoft.DocAsCode.Build.Engine/DocumentBuildParameters.cs
@@ -40,7 +40,7 @@ public sealed class DocumentBuildParameters : IBuildParameters
 
         public int MaxHttpParallelism { get; set; }
 
-        public string MarkdownEngineName { get; set; } = "dfm";
+        public string MarkdownEngineName { get; set; } = "markdig";
 
         public ImmutableDictionary<string, object> MarkdownEngineParameters { get; set; } = ImmutableDictionary<string, object>.Empty;
 

diff --git a/src/Microsoft.DocAsCode.Build.Engine/DocumentBuilder.cs b/src/Microsoft.DocAsCode.Build.Engine/DocumentBuilder.cs
@@ -239,13 +239,12 @@ IMarkdownServiceProvider GetMarkdownServiceProvider()
                 }
 
                 var result = CompositionContainer.GetExport<IMarkdownServiceProvider>(_container, markdownEngineName);
-                    if (result == null)
-                    {
+                if (result == null)
+                {
                     Logger.LogError($"Unable to find markdown engine: {markdownEngineName}");
                     throw new DocfxException($"Unable to find markdown engine: {markdownEngineName}");
-                    }
-                Logger.LogInfo($"Markdown engine is {markdownEngineName}", code: "MarkdownEngineName");
-                    return result;
+                }
+                return result;
             }
         }
 

diff --git a/src/Microsoft.DocAsCode.Build.Engine/HostService.cs b/src/Microsoft.DocAsCode.Build.Engine/HostService.cs
@@ -146,8 +146,8 @@ private MarkupResult MarkupCore(string markdown, FileAndType ft, bool omitParse,
         {
             try
             {
-                var mr = MarkdownService is MarkdigMarkdownService
-                    ? MarkdownService.Markup(markdown, ft.File, enableValidation)
+                var mr = MarkdownService is MarkdigMarkdownService markdig
+                    ? markdig.Markup(markdown, ft.File, enableValidation, ft.Type is DocumentType.Overwrite)
                     : MarkdownService.Markup(markdown, ft.File);
                 if (omitParse)
                 {

diff --git a/src/Microsoft.DocAsCode.Build.Engine/MarkdownProviders/DfmJsonTokenTreeServiceProvider.cs b/src/Microsoft.DocAsCode.Build.Engine/MarkdownProviders/DfmJsonTokenTreeServiceProvider.cs
@@ -56,11 +56,6 @@ public MarkupResult Markup(string src, string path)
                 }
                 return result;
             }
-
-            public MarkupResult Markup(string src, string path, bool enableValidation)
-            {
-                throw new NotImplementedException();
-            }
         }
     }
 }
diff --git a/src/Microsoft.DocAsCode.Build.Engine/MarkdownProviders/DfmServiceProvider.cs b/src/Microsoft.DocAsCode.Build.Engine/MarkdownProviders/DfmServiceProvider.cs
@@ -126,11 +126,6 @@ public MarkupResult Markup(string src, string path)
                 return result;
             }
 
-            public MarkupResult Markup(string src, string path, bool enableValidation)
-            {
-                throw new NotImplementedException();
-            }
-
             public void Dispose()
             {
                 (Renderer as IDisposable)?.Dispose();

diff --git a/src/Microsoft.DocAsCode.Build.Engine/MarkdownProviders/GfmServiceProvider.cs b/src/Microsoft.DocAsCode.Build.Engine/MarkdownProviders/GfmServiceProvider.cs
@@ -32,11 +32,6 @@ public MarkupResult Markup(string src, string path)
                     Html = html,
                 };
             }
-
-            public MarkupResult Markup(string src, string path, bool enableValidation)
-            {
-                throw new NotImplementedException();
-            }
         }
     }
 }
diff --git a/src/Microsoft.DocAsCode.Build.Engine/MarkdownProviders/JsonTokenTreeServiceProvider.cs b/src/Microsoft.DocAsCode.Build.Engine/MarkdownProviders/JsonTokenTreeServiceProvider.cs
@@ -38,11 +38,6 @@ public MarkupResult Markup(string src, string path)
                     Html = $"{{\"name\":\"0>0>markdown\",\"children\":[{json}]}}",
                 };
             }
-
-            public MarkupResult Markup(string src, string path, bool enableValidation)
-            {
-                throw new NotImplementedException();
-            }
         }
     }
 }
diff --git a/src/Microsoft.DocAsCode.MarkdigEngine/MarkdigMarkdownService.cs b/src/Microsoft.DocAsCode.MarkdigEngine/MarkdigMarkdownService.cs
@@ -47,10 +47,10 @@ public MarkdigMarkdownService(
 
         public MarkupResult Markup(string content, string filePath)
         {
-            return Markup(content, filePath, false);
+            return Markup(content, filePath, false, false);
         }
 
-        public MarkupResult Markup(string content, string filePath, bool enableValidation)
+        public MarkupResult Markup(string content, string filePath, bool enableValidation, bool multipleYamlHeader)
         {
             if (content == null)
             {
@@ -62,7 +62,7 @@ public MarkupResult Markup(string content, string filePath, bool enableValidatio
                 throw new ArgumentException("file path can't be null or empty.");
             }
 
-            var pipeline = CreateMarkdownPipeline(isInline: false, enableValidation: enableValidation);
+            var pipeline = CreateMarkdownPipeline(isInline: false, enableValidation, multipleYamlHeader);
 
             using (InclusionContext.PushFile((RelativePath)filePath))
             {
@@ -137,7 +137,7 @@ public MarkupResult Render(MarkdownDocument document, bool isInline)
             }
         }
 
-        private MarkdownPipeline CreateMarkdownPipeline(bool isInline, bool enableValidation)
+        private MarkdownPipeline CreateMarkdownPipeline(bool isInline, bool enableValidation, bool multipleYamlHeader = false)
         {
             object enableSourceInfoObj = null;
             _parameters?.Extensions?.TryGetValue(Constants.EngineProperties.EnableSourceInfo, out enableSourceInfoObj);
@@ -147,7 +147,7 @@ private MarkdownPipeline CreateMarkdownPipeline(bool isInline, bool enableValida
             var builder = new MarkdownPipelineBuilder();
 
             builder.UseDocfxExtensions(_context);
-            builder.Extensions.Insert(0, new YamlHeaderExtension(_context));
+            builder.Extensions.Insert(0, new YamlHeaderExtension(_context) { AllowInMiddleOfDocument = multipleYamlHeader });
 
             if (enableSourceInfo)
             {

diff --git a/src/Microsoft.DocAsCode.MarkdigEngine/YamlHeader/YamlHeaderExtension.cs b/src/Microsoft.DocAsCode.MarkdigEngine/YamlHeader/YamlHeaderExtension.cs
@@ -13,6 +13,8 @@ public class YamlHeaderExtension : IMarkdownExtension
     {
         private readonly MarkdownContext _context;
 
+        public bool AllowInMiddleOfDocument { get; init; }
+
         public YamlHeaderExtension(MarkdownContext context)
         {
             _context = context;
@@ -23,7 +25,7 @@ public void Setup(MarkdownPipelineBuilder pipeline)
             if (!pipeline.BlockParsers.Contains<YamlFrontMatterParser>())
             {
                 // Insert the YAML parser before the thematic break parser, as it is also triggered on a --- dash
-                pipeline.BlockParsers.InsertBefore<ThematicBreakParser>(new YamlFrontMatterParser());
+                pipeline.BlockParsers.InsertBefore<ThematicBreakParser>(new YamlFrontMatterParser { AllowInMiddleOfDocument = AllowInMiddleOfDocument });
             }
         }
 

diff --git a/src/Microsoft.DocAsCode.MarkdigEngine/YamlHeader/YamlHeaderRenderer.cs b/src/Microsoft.DocAsCode.MarkdigEngine/YamlHeader/YamlHeaderRenderer.cs
@@ -1,52 +1,52 @@
-// Copyright (c) Microsoft. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-namespace Microsoft.DocAsCode.MarkdigEngine.Extensions
-{
-    using System;
-    using System.Collections.Generic;
-    using System.IO;
-    using System.Net;
-
-    using Markdig.Extensions.Yaml;
-    using Markdig.Renderers;
-    using Markdig.Renderers.Html;
-    using Microsoft.DocAsCode.Common;
-
-    public class YamlHeaderRenderer : HtmlObjectRenderer<YamlFrontMatterBlock>
-    {
-        private readonly MarkdownContext _context;
-
-        public YamlHeaderRenderer(MarkdownContext context)
-        {
-            _context = context;
-        }
-
-        protected override void Write(HtmlRenderer renderer, YamlFrontMatterBlock obj)
-        {
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.DocAsCode.MarkdigEngine.Extensions
+{
+    using System;
+    using System.Collections.Generic;
+    using System.IO;
+    using System.Net;
+
+    using Markdig.Extensions.Yaml;
+    using Markdig.Renderers;
+    using Markdig.Renderers.Html;
+    using Microsoft.DocAsCode.Common;
+
+    public class YamlHeaderRenderer : HtmlObjectRenderer<YamlFrontMatterBlock>
+    {
+        private readonly MarkdownContext _context;
+
+        public YamlHeaderRenderer(MarkdownContext context)
+        {
+            _context = context;
+        }
+
+        protected override void Write(HtmlRenderer renderer, YamlFrontMatterBlock obj)
+        {
             if (InclusionContext.IsInclude)
             {
                 return;
-            }
-
-            var content = obj.Lines.ToString();
-            try
-            {
-                using StringReader reader = new StringReader(content);
-                var result = YamlUtility.Deserialize<Dictionary<string, object>>(reader);
-                if (result != null)
-                {
-                    renderer.Write("<yamlheader").Write($" start=\"{obj.Line + 1}\" end=\"{obj.Line + obj.Lines.Count + 2}\"");
-                    renderer.WriteAttributes(obj).Write(">");
-                    renderer.Write(WebUtility.HtmlEncode(obj.Lines.ToString()));
-                    renderer.Write("</yamlheader>");
-                }
-            }
-            catch (Exception ex)
-            {
-                // not a valid ymlheader, do nothing
-                _context.LogWarning("invalid-yaml-header", ex.Message, obj);
-            }
-        }
-    }
-}
+            }
+
+            var content = obj.Lines.ToString();
+            try
+            {
+                using StringReader reader = new StringReader(content);
+                var result = YamlUtility.Deserialize<Dictionary<string, object>>(reader);
+                if (result != null)
+                {
+                    renderer.Write("<yamlheader").Write($" start=\"{obj.Line + 1}\" end=\"{obj.Line + obj.Lines.Count + 2}\"");
+                    renderer.WriteAttributes(obj).Write(">");
+                    renderer.Write(WebUtility.HtmlEncode(obj.Lines.ToString()));
+                    renderer.Write("</yamlheader>");
+                }
+            }
+            catch (Exception ex)
+            {
+                // not a valid ymlheader, do nothing
+                _context.LogWarning("invalid-yaml-header", ex.Message, obj);
+            }
+        }
+    }
+}
diff --git a/src/Microsoft.DocAsCode.Plugins/IMarkdownService.cs b/src/Microsoft.DocAsCode.Plugins/IMarkdownService.cs
@@ -8,7 +8,5 @@ public interface IMarkdownService
         string Name { get; }
 
         MarkupResult Markup(string src, string path);
-
-        MarkupResult Markup(string src, string path, bool enableValidation);
     }
 }
diff --git a/test/Microsoft.DocAsCode.Build.Common.Tests/MarkdownReaderTest.cs b/test/Microsoft.DocAsCode.Build.Common.Tests/MarkdownReaderTest.cs
@@ -13,6 +13,7 @@ namespace Microsoft.DocAsCode.Build.Common.Tests
     using Microsoft.DocAsCode.Dfm;
     using Microsoft.DocAsCode.Build.Engine;
     using Microsoft.DocAsCode.Plugins;
+    using Microsoft.DocAsCode.MarkdigEngine;
 
     public class MarkdownReaderTest
     {
@@ -33,7 +34,7 @@ public void TestReadMarkdownAsOverwrite()
             File.WriteAllText(fullPath, content);
             var host = new HostService(null, Enumerable.Empty<FileModel>())
             {
-                MarkdownService = new DfmServiceProvider().CreateMarkdownService(new MarkdownServiceParameters {BasePath = string.Empty}),
+                MarkdownService = new MarkdigServiceProvider().CreateMarkdownService(new MarkdownServiceParameters {BasePath = string.Empty}),
                 SourceFiles = ImmutableDictionary.Create<string, FileAndType>()
             };
 
@@ -43,7 +44,7 @@ public void TestReadMarkdownAsOverwrite()
             Assert.Single(results);
             Assert.Equal("Test", results[0].Uid);
             Assert.Equal("Hello", results[0].Metadata["remarks"]);
-            Assert.Equal("<p sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"6\" sourceendlinenumber=\"6\">This is unit test!</p>\n", results[0].Conceptual);
+            Assert.Equal("\n<p sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"6\">This is unit test!</p>\n", results[0].Conceptual);
             File.Delete(fileName);
 
             // Test conceptual content between two yamlheader
@@ -66,11 +67,10 @@ This is unit test!
             Assert.Equal("Test1", results[0].Uid);
             Assert.Equal("Test2", results[1].Uid);
             Assert.Equal("Hello", results[0].Metadata["remarks"]);
-            Assert.Equal("<p sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"5\" sourceendlinenumber=\"5\">This is unit test!</p>\n", results[0].Conceptual);
+            Assert.Equal("\n<p sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"5\">This is unit test!</p>\n", results[0].Conceptual);
             Assert.Equal(string.Empty, results[1].Conceptual);
             File.Delete(fileName);
 
-            // Invalid yamlheader is not supported
             content = @"---
 uid: Test1
 remarks: Hello
@@ -85,10 +85,12 @@ This is unit test!
             File.WriteAllText(fileName, content);
             results = MarkdownReader.ReadMarkdownAsOverwrite(host, ft).ToList();
             Assert.NotNull(results);
-            Assert.Single(results);
+            Assert.Equal(2, results.Count);
             Assert.Equal("Test1", results[0].Uid);
             Assert.Equal("Hello", results[0].Metadata["remarks"]);
-            Assert.Equal("<h2 id=\"this-is-unit-test\" sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"5\" sourceendlinenumber=\"6\">This is unit test!</h2>\n<h2 id=\"uid-test2\" sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"7\" sourceendlinenumber=\"8\">uid: Test2</h2>\n", results[0].Conceptual);
+            Assert.Equal("\n<p sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"5\">This is unit test!</p>\n", results[0].Conceptual);
+            Assert.Equal("Test2", results[1].Uid);
+            Assert.Equal("", results[1].Conceptual);
             File.Delete(fileName);
 
             // Test conceptual content with extra empty line between two yamlheader
@@ -114,7 +116,7 @@ This is unit test!
             Assert.Equal("Test1", results[0].Uid);
             Assert.Equal("Test2", results[1].Uid);
             Assert.Equal("Hello", results[0].Metadata["remarks"]);
-            Assert.Equal("<p sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"7\" sourceendlinenumber=\"7\">This is unit test!</p>\n", results[0].Conceptual);
+            Assert.Equal("\n<p sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"7\">This is unit test!</p>\n", results[0].Conceptual);
             Assert.Equal(string.Empty, results[1].Conceptual);
             File.Delete(fileName);
 
@@ -127,7 +129,7 @@ This is unit test!
             Assert.Single(results);
             Assert.Equal("Test", results[0].Uid);
             Assert.Equal("Hello", results[0].Metadata["remarks"]);
-            Assert.Equal("<p sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"5\" sourceendlinenumber=\"5\">This is unit test!</p>\n", results[0].Conceptual);
+            Assert.Equal("\n<p sourcefile=\"ut_ReadMarkdownAsOverwrite.md\" sourcestartlinenumber=\"5\">This is unit test!</p>\n", results[0].Conceptual);
             File.Delete(fileName);
 
             // Test link to files and Uids in overwrite document
@@ -177,12 +179,15 @@ [Not exist link2](link2.md)
             Assert.Equal(5, uidLinkSource[0].LineNumber);
             Assert.Equal(fileName, uidLinkSource[0].SourceFile);
             Assert.Equal("NotExistUid", uidLinkSource[0].Target);
-            Assert.Equal(@"<p sourcefile=""ut_ReadMarkdownAsOverwrite.md"" sourcestartlinenumber=""5"" sourceendlinenumber=""5""><xref href=""NotExistUid"" data-throw-if-not-resolved=""False"" data-raw-source=""@NotExistUid"" sourcefile=""ut_ReadMarkdownAsOverwrite.md"" sourcestartlinenumber=""5"" sourceendlinenumber=""5""></xref></p>
-<p sourcefile=""ut_ReadMarkdownAsOverwrite.md"" sourcestartlinenumber=""7"" sourceendlinenumber=""8""><a href=""link.md"" data-raw-source=""[Not exist link](link.md)"" sourcefile=""ut_ReadMarkdownAsOverwrite.md"" sourcestartlinenumber=""7"" sourceendlinenumber=""7"">Not exist link</a>
-<a href=""link2.md"" data-raw-source=""[Not exist link2](link2.md)"" sourcefile=""ut_ReadMarkdownAsOverwrite.md"" sourcestartlinenumber=""8"" sourceendlinenumber=""8"">Not exist link2</a></p>
-<p sourcefile=""ut_ReadMarkdownAsOverwrite.md"" sourcestartlinenumber=""10"" sourceendlinenumber=""10"">This is unit test!</p>
-".Replace("\r\n", "\n"),
-                results[0].Conceptual);
+            Assert.Equal(
+                """
+                <p sourcefile="ut_ReadMarkdownAsOverwrite.md" sourcestartlinenumber="5"><xref href="NotExistUid" data-throw-if-not-resolved="False" data-raw-source="@NotExistUid" sourcefile="ut_ReadMarkdownAsOverwrite.md" sourcestartlinenumber="5"></xref></p>
+                <p sourcefile="ut_ReadMarkdownAsOverwrite.md" sourcestartlinenumber="7"><a href="link.md" sourcefile="ut_ReadMarkdownAsOverwrite.md" sourcestartlinenumber="7">Not exist link</a>
+                <a href="link2.md" sourcefile="ut_ReadMarkdownAsOverwrite.md" sourcestartlinenumber="8">Not exist link2</a></p>
+                <p sourcefile="ut_ReadMarkdownAsOverwrite.md" sourcestartlinenumber="10">This is unit test!</p>
+                """,
+                results[0].Conceptual.Trim(),
+                ignoreLineEndingDifferences: true);
             File.Delete(fileName);
         }
     }