Merge pull request #37 from razeware/WEB-3588

WEB-3588: Switching the markdown engine

Gemfile

@@ -13,7 +13,7 @@ gem 'cli-ui', '~> 1.3'
 gem 'thor', '~> 1.0', '>= 1.0.1'
 
 # Markdown processing
-gem 'redcarpet', '~> 3.5'
+gem 'commonmarker'
 
 # HTTP Client
 gem 'faraday'

Gemfile.lock

@@ -41,6 +41,8 @@ GEM
     benchmark (0.1.0)
     cli-ui (1.3.0)
     coderay (1.1.3)
+    commonmarker (0.21.0)
+      ruby-enum (~> 0.5)
     concurrent-ruby (1.1.6)
     debase (0.2.4.1)
       debase-ruby_core_source (>= 0.10.2)
@@ -115,24 +117,25 @@ GEM
     rbnacl (7.1.1)
       ffi
     rchardet (1.8.0)
-    redcarpet (3.5.0)
     regexp_parser (1.7.1)
     reverse_markdown (2.0.0)
       nokogiri
     rexml (3.2.4)
-    rubocop (0.89.0)
+    rubocop (0.89.1)
       parallel (~> 1.10)
       parser (>= 2.7.1.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.7)
       rexml
-      rubocop-ast (>= 0.1.0, < 1.0)
+      rubocop-ast (>= 0.3.0, < 1.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
     rubocop-ast (0.3.0)
       parser (>= 2.7.1.4)
     ruby-debug-ide (0.7.2)
       rake (>= 0.8.1)
+    ruby-enum (0.8.0)
+      i18n
     ruby-progressbar (1.10.1)
     ruby2_keywords (0.0.2)
     sassc (2.4.0)
@@ -147,7 +150,7 @@ GEM
       rack-protection (= 2.0.8.1)
       tilt (~> 2.0)
     slack-notifier (2.3.2)
-    solargraph (0.39.13)
+    solargraph (0.39.15)
       backport (~> 1.1)
       benchmark
       bundler (>= 1.17.2)
@@ -178,6 +181,7 @@ DEPENDENCIES
   activesupport (~> 6.0)
   aws-sdk-s3 (~> 1.64)
   cli-ui (~> 1.3)
+  commonmarker
   concurrent-ruby (~> 1.1)
   debase (~> 0.2.4.1)
   faraday
@@ -188,7 +192,6 @@ DEPENDENCIES
   octokit!
   rack-livereload
   rbnacl
-  redcarpet (~> 3.5)
   rubocop (~> 0.81)
   ruby-debug-ide (~> 0.7.2)
   sassc

app/lib/image_provider/markdown_image_extractor.rb

@@ -3,47 +3,30 @@
 module ImageProvider
   # Takes a markdown file, and returns all the images URLs
   class MarkdownImageExtractor
-    # Process markdown, and extract a list of images
-    class MarkdownRenderer < Redcarpet::Render::Base
-      attr_reader :images
-
-      def initialize
-        super
-        @images = []
-      end
-
-      def reset
-        @images = []
-      end
-
-      def image(link, _title, _alt_text)
-        images << link
-        nil
-      end
-    end
-
     include Util::PathExtraction
 
     def self.images_from(file)
       new(file: file).images
     end
 
     def images
-      md_renderer.reset
-      renderer.render(markdown)
-      md_renderer.images.map { |path| { relative_path: cleanpath(path), absolute_path: cleanpath(apply_path(path)) } }
+      [].tap do |images|
+        doc.walk do |node|
+          images << image_record(node.url) if node.type == :image
+        end
+      end
     end
 
-    def cleanpath(path)
-      Pathname.new(path).cleanpath.to_s
+    def image_record(url)
+      { relative_path: cleanpath(url), absolute_path: cleanpath(apply_path(url)) }
     end
 
-    def renderer
-      @renderer ||= Redcarpet::Markdown.new(md_renderer)
+    def cleanpath(path)
+      Pathname.new(path).cleanpath.to_s
     end
 
-    def md_renderer
-      @md_renderer ||= MarkdownRenderer.new
+    def doc
+      @doc ||= CommonMarker.render_doc(markdown)
     end
 
     def markdown

app/lib/renderer/markdown_file_renderer.rb

@@ -4,6 +4,7 @@ module Renderer
   # Read a file and render the markdown
   class MarkdownFileRenderer
     include Util::Logging
+    include Parser::FrontmatterMetadataFinder
 
     attr_reader :path
     attr_reader :image_provider
@@ -15,27 +16,43 @@ def initialize(path:, image_provider: nil)
 
     def render
       logger.debug 'MarkdownFileRenderer::render'
-      redcarpet.render(raw_content)
+      remove_h1(doc)
+      rw_renderer.render(doc)
+    end
+
+    def rw_renderer
+      @rw_renderer ||= Renderer::RWMarkdownRenderer.new(
+        options: %i[TABLE_PREFER_STYLE_ATTRIBUTES],
+        extensions: %i[table strikethrough autolink],
+        image_provider: image_provider,
+        root_path: root_directory
+      )
     end
 
     def raw_content
       @raw_content ||= File.read(path)
     end
 
-    def redcarpet_renderer
-      @redcarpet_renderer ||= RWMarkdownRenderer.new(with_toc_data: true,
-                                                     image_provider: image_provider,
-                                                     root_path: root_directory)
+    def preproccessed_markdown
+      @preproccessed_markdown ||= begin
+        removing_pagesetting_notation = raw_content.gsub(/\$\[=[=sp]=\]/, '')
+        without_metadata(removing_pagesetting_notation.each_line)
+      end
+    end
+
+    def doc
+      @doc ||= CommonMarker.render_doc(
+        preproccessed_markdown,
+        %i[SMART STRIKETHROUGH_DOUBLE_TILDE],
+        %i[table strikethrough autolink]
+      )
     end
 
-    def redcarpet
-      @redcarpet ||= Redcarpet::Markdown.new(redcarpet_renderer,
-                                             fenced_code_blocks: true,
-                                             disable_indented_code_blocks: true,
-                                             autolink: true,
-                                             strikethrough: true,
-                                             tables: true,
-                                             hightlight: true)
+    def remove_h1(document)
+      document.walk do |node|
+        node.delete if node.type == :header && node.header_level.to_i == 1
+      end
+      document
     end
 
     def root_directory

app/lib/renderer/markdown_string_renderer.rb

@@ -13,21 +13,15 @@ def initialize(content:)
 
     def render
       logger.debug 'MarkdownStringRenderer::render'
-      redcarpet.render(content)
-    end
-
-    def redcarpet_renderer
-      @redcarpet_renderer ||= RWMarkdownRenderer.new(with_toc_data: true)
-    end
-
-    def redcarpet
-      @redcarpet ||= Redcarpet::Markdown.new(redcarpet_renderer,
-                                             fenced_code_blocks: true,
-                                             disable_indented_code_blocks: true,
-                                             autolink: true,
-                                             strikethrough: true,
-                                             tables: true,
-                                             hightlight: true)
+      doc = CommonMarker.render_doc(
+        content,
+        %i[SMART STRIKETHROUGH_DOUBLE_TILDE],
+        %i[table strikethrough autolink]
+      )
+      doc.to_html(
+        %i[TABLE_PREFER_STYLE_ATTRIBUTES],
+        %i[table strikethrough autolink]
+      )
     end
   end
 end

app/lib/renderer/rw_markdown_renderer.rb

@@ -2,45 +2,32 @@
 
 module Renderer
   # Custom implementation of a markdown renderer for RW books
-  class RWMarkdownRenderer < Redcarpet::Render::HTML
-    include Redcarpet::Render::SmartyPants
-    include Parser::FrontmatterMetadataFinder
+  class RWMarkdownRenderer < CommonMarker::HtmlRenderer
     include Renderer::ImageAttributes
     include Util::Logging
 
     attr_reader :root_path
 
-    def initialize(attributes = {})
+    def initialize(options: :DEFAULT, extensions: [], image_provider:, root_path:)
       logger.debug 'RWMarkdownRenderer::initialize'
-      super
-      @image_provider = attributes[:image_provider]
-      @root_path = attributes[:root_path]
+      super(options: options, extensions: extensions)
+      @image_provider = image_provider
+      @root_path = root_path
     end
 
-    def header(text, header_level)
-      return nil if header_level == 1
+    def image(node)
+      return super(node) if image_provider.blank?
 
-      "<h#{header_level}>#{text}</h#{header_level}>"
-    end
-
-    def image(link, title, alt_text)
-      return %(<img src="#{link}" alt="#{alt_text}" title="#{title}" />) if image_provider.blank?
-
-      %(
-        <figure title="#{title}" class="#{class_list(alt_text)}">
-          <picture>
-            <img src="#{src(link)}" alt="#{title}" title="#{title}">
-            <source srcset="#{srcset(link)}">
-          </picture>
-          <figcaption>#{title}</figcaption>
-        </figure>
-      )
-    end
+      title = node.title.present? ? escape_html(node.title) : ''
+      classes = class_list(node.each.select { |child| child.type == :text }.map { |child| escape_html(child.string_content) }.join(' '))
 
-    def preprocess(full_document)
-      logger.debug 'RWMarkdownRenderer::preprocess'
-      removing_pagesetting_notation = full_document.gsub(/\$\[=[=sp]=\]/, '')
-      without_metadata(removing_pagesetting_notation.each_line)
+      out('<figure title="', title, '"', ' class="', classes, '">')
+      out('  <picture>')
+      out('    <img src="', src(node.url), '" alt="', title, '" title="', title, '">')
+      out('    <source srcset="', srcset(node.url), '">')
+      out('  </picture>')
+      out('  <figcaption>', title, '</figcaption>')
+      out('</figure>')
     end
   end
 end