Merge pull request #920 from metanorma/fix/yaml-no-xml

Fix/yaml no xml

Gemfile.devel

@@ -0,0 +1 @@
+gem "isodoc", git: "https://github.com/metanorma/isodoc", branch: "fix/yaml-no-xml"

lib/metanorma/standoc/cleanup.rb

@@ -54,6 +54,7 @@ def cleanup(xmldoc)
         normref_cleanup(xmldoc)
         biblio_cleanup(xmldoc)
         reference_names(xmldoc)
+        terms_terms_cleanup(xmldoc) # feeds: boilerplate_cleanup
         asciimath_cleanup(xmldoc) # feeds: mathml_cleanup, termdef_cleanup,
         # symbols_cleanup
         symbols_cleanup(xmldoc) # feeds: termdef_cleanup
@@ -77,9 +78,10 @@ def cleanup(xmldoc)
         docidentifier_cleanup(xmldoc) # feeds: bibdata_cleanup
         ext_contributor_cleanup(xmldoc) # feeds: bibdata_cleanup
         ext_dochistory_cleanup(xmldoc) # feeds: bibdata_cleanup
-        bibdata_cleanup(xmldoc)
+        bibdata_cleanup(xmldoc) # feeds: boilerplate_cleanup
+        boilerplate_cleanup(xmldoc) # feeds: xref_cleanup for new <<>> introduced
+        xref_cleanup(xmldoc)
         svgmap_cleanup(xmldoc) # feeds: img_cleanup
-        boilerplate_cleanup(xmldoc)
         toc_cleanup(xmldoc)
         smartquotes_cleanup(xmldoc)
         linebreak_cleanup(xmldoc)

lib/metanorma/standoc/cleanup_boilerplate.rb

@@ -1,41 +1,8 @@
+require_relative "cleanup_terms_boilerplate"
+
 module Metanorma
   module Standoc
     module Cleanup
-      def external_terms_boilerplate(sources)
-        e = @i18n.external_terms_boilerplate
-        @i18n.l10n(e.gsub(/%(?=\p{P}|\p{Z}|$)/, sources || "???"),
-                   @lang, @script, @locale)
-      end
-
-      def internal_external_terms_boilerplate(sources)
-        e = @i18n.internal_external_terms_boilerplate
-        @i18n.l10n(e.gsub(/%(?=\p{P}|\p{Z}|$)/, sources || "??"),
-                   @lang, @script)
-      end
-
-      def term_defs_boilerplate(div, source, term, _preface, isodoc)
-        a = @i18n.term_def_boilerplate and div.next = a
-        source.each do |s|
-          @anchors[s["bibitemid"]] or
-            @log.add("Crossreferences", nil,
-                     "term source #{s['bibitemid']} not referenced", severity: 1)
-        end
-        a = if source.empty? && term.nil? then @i18n.no_terms_boilerplate
-            else term_defs_boilerplate_cont(source, term, isodoc)
-            end and div.next = a
-      end
-
-      def term_defs_boilerplate_cont(src, term, isodoc)
-        sources = isodoc.sentence_join(src.map do |s|
-          %{<eref bibitemid="#{s['bibitemid']}"/>}
-        end)
-        if src.empty? then @i18n.internal_terms_boilerplate
-        elsif term.nil? then external_terms_boilerplate(sources)
-        else
-          internal_external_terms_boilerplate(sources)
-        end
-      end
-
       def norm_ref_preface(ref)
         ins = norm_ref_boilerplate_insert_location(ref)
         ins2 = norm_ref_process_boilerplate_note(ref)
@@ -45,7 +12,7 @@ def norm_ref_preface(ref)
           %w(references bibitem).include? e.name
         end
         pref = refs.empty? ? @i18n.norm_empty_pref : @i18n.norm_with_refs_pref
-        ins.next = "<p>#{pref}</p>"
+        ins.next = boilerplate_snippet_convert(pref)
       end
 
       def norm_ref_process_boilerplate_note(ref)
@@ -69,16 +36,15 @@ def norm_ref_boilerplate_insert_location(ref)
         ref.at("./title")
       end
 
-      TERM_CLAUSE =
-        "//sections//terms[not(.//ancestor::clause[@type = 'terms'])] | " \
-        "//sections/clause[descendant::terms][@type = 'terms'] | " \
-        "//sections/clause[not(@type = 'terms')]//terms".freeze
-
       NORM_REF =
         "//bibliography/references[@normative = 'true'][not(@hidden)] | " \
         "//bibliography/clause[.//references[@normative = 'true']]".freeze
 
       def boilerplate_isodoc(xmldoc)
+        # prevent infinite recursion of asciidoc boilerplate processing
+        # in termdef_boilerplate_insert and initial_boilerplate
+        xmldoc.at("//metanorma-extension/semantic-metadata/" \
+                  "headless[text() = 'true']") and return nil
         x = xmldoc.dup
         x.root.add_namespace(nil, self.class::XML_NAMESPACE)
         xml = Nokogiri::XML(x.to_xml)
@@ -87,14 +53,6 @@ def boilerplate_isodoc(xmldoc)
         @isodoc
       end
 
-      def termdef_boilerplate_cleanup(xmldoc)
-        # termdef_remove_initial_paras(xmldoc)
-      end
-
-      def termdef_remove_initial_paras(xmldoc)
-        xmldoc.xpath("//terms/p | //terms/ul").each(&:remove)
-      end
-
       def unwrap_boilerplate_clauses(xmldoc, xpath)
         xmldoc.xpath(xpath).each do |f|
           f.xpath(".//clause[@type = 'boilerplate'] | " \
@@ -105,51 +63,8 @@ def unwrap_boilerplate_clauses(xmldoc, xpath)
         end
       end
 
-      def termdef_boilerplate_insert(xmldoc, isodoc, once = false)
-        if once
-          f = termdef_boilerplate_insert_location(xmldoc) and
-            termdef_boilerplate_insert1(f, xmldoc, isodoc)
-        else
-          xmldoc.xpath(self.class::TERM_CLAUSE).each do |f|
-            termdef_boilerplate_insert1(f, xmldoc, isodoc)
-          end
-        end
-      end
-
-      def termdef_boilerplate_insert_location(xmldoc)
-        f = xmldoc.at(self.class::TERM_CLAUSE)
-        root = xmldoc.at("//sections/terms | //sections/clause[@type = 'terms']")
-        if f && root && f["id"] != root["id"]
-          f = termdef_boilerplate_climb_up(f, root)
-        elsif !f && root then f = root
-        end
-        f
-      end
-
-      def termdef_boilerplate_climb_up(clause, container)
-        container.at(".//*[@id = '#{clause['id']}']") or return clause
-        while (n = clause.parent)
-          n.at(".//definitions") and break
-          clause = n
-          n["id"] == container["id"] and break
-        end
-        clause
-      end
-
-      def termdef_boilerplate_insert1(sect, xmldoc, isodoc)
-        ins = sect.at("./title")
-        if (ins2 = sect.at("./clause[@type = 'boilerplate'] | " \
-                "./note[@type = 'boilerplate']"))
-          ins2.text.strip.downcase == "(default)" or return
-          ins2.children = " "
-          ins = ins2.children.first
-        end
-        term_defs_boilerplate(ins, xmldoc.xpath(".//termdocsource"),
-                              sect.at(".//term"), sect.at(".//p"), isodoc)
-      end
-
       def boilerplate_cleanup(xmldoc)
-        isodoc = boilerplate_isodoc(xmldoc)
+        isodoc = boilerplate_isodoc(xmldoc) or return
         termdef_boilerplate_cleanup(xmldoc)
         termdef_boilerplate_insert(xmldoc, isodoc)
         unwrap_boilerplate_clauses(xmldoc, self.class::TERM_CLAUSE)
@@ -245,6 +160,7 @@ def boilerplate_xml_cleanup(xml)
           /^_\d+$/.match?(n["id"]) and
             n["id"] = "_#{UUIDTools::UUID.random_create}"
         end
+        xml
       end
 
       def boilerplate_top_elements(xml)

lib/metanorma/standoc/cleanup_terms.rb

@@ -73,11 +73,14 @@ def termdocsource_cleanup(xmldoc)
         xmldoc.xpath("//termdocsource").each { |s| f.previous = s.remove }
       end
 
-      def term_children_cleanup(xmldoc)
+      def terms_terms_cleanup(xmldoc)
         xmldoc.xpath("//terms[terms][not(term)]").each do |t|
           t.name = "clause"
           t["type"] = "terms"
         end
+      end
+
+      def term_children_cleanup(xmldoc)
         xmldoc.xpath("//term").each do |t|
           %w(termnote termexample termsource term).each do |w|
             t.xpath("./#{w}").each { |n| t << n.remove }

lib/metanorma/standoc/cleanup_terms_boilerplate.rb

@@ -0,0 +1,106 @@
+module Metanorma
+  module Standoc
+    module Cleanup
+      def external_terms_boilerplate(sources)
+        e = @i18n.external_terms_boilerplate
+        e.gsub(/%(?=\p{P}|\p{Z}|$)/, sources || "???")
+      end
+
+      def internal_external_terms_boilerplate(sources)
+        e = @i18n.internal_external_terms_boilerplate
+        e.gsub(/%(?=\p{P}|\p{Z}|$)/, sources || "??")
+      end
+
+      def boilerplate_snippet_convert(adoc)
+        ret = boilerplate_xml_cleanup(adoc2xml(adoc, backend.to_sym))
+        @i18n.l10n(ret.children.to_xml, @lang, @script)
+      end
+
+      def term_defs_boilerplate(div, source, term, _preface, isodoc)
+        verify_term_defs_source(source)
+        a = @i18n.term_def_boilerplate and
+          div.next = boilerplate_snippet_convert(a)
+        a = if source.empty? && term.nil? then @i18n.no_terms_boilerplate
+            else term_defs_boilerplate_cont(source, term, isodoc)
+            end
+        a and div.next = boilerplate_snippet_convert(a)
+      end
+
+      def verify_term_defs_source(source)
+        source.each do |s|
+          @anchors[s["bibitemid"]] or
+            @log.add("Crossreferences", nil,
+                     "term source #{s['bibitemid']} not referenced",
+                     severity: 1)
+        end
+      end
+
+      def term_defs_boilerplate_cont(src, term, isodoc)
+        sources = isodoc.sentence_join(src.map do |s|
+          %{<<#{s['bibitemid']}>>}
+        end)
+        if src.empty? then @i18n.internal_terms_boilerplate
+        elsif term.nil? then external_terms_boilerplate(sources)
+        else
+          internal_external_terms_boilerplate(sources)
+        end
+      end
+
+      TERM_CLAUSE =
+        "//sections//terms[not(.//ancestor::clause[@type = 'terms'])] | " \
+        "//sections/clause[descendant::terms][@type = 'terms'] | " \
+        "//sections/clause[not(@type = 'terms')]//terms".freeze
+
+      def termdef_boilerplate_cleanup(xmldoc)
+        # termdef_remove_initial_paras(xmldoc)
+      end
+
+      def termdef_remove_initial_paras(xmldoc)
+        xmldoc.xpath("//terms/p | //terms/ul").each(&:remove)
+      end
+
+      def termdef_boilerplate_insert(xmldoc, isodoc, once = false)
+        if once
+          f = termdef_boilerplate_insert_location(xmldoc) and
+            termdef_boilerplate_insert1(f, xmldoc, isodoc)
+        else
+          xmldoc.xpath(self.class::TERM_CLAUSE).each do |f|
+            termdef_boilerplate_insert1(f, xmldoc, isodoc)
+          end
+        end
+      end
+
+      def termdef_boilerplate_insert_location(xmldoc)
+        f = xmldoc.at(self.class::TERM_CLAUSE)
+        root = xmldoc.at("//sections/terms | //sections/clause[@type = 'terms']")
+        if f && root && f["id"] != root["id"]
+          f = termdef_boilerplate_climb_up(f, root)
+        elsif !f && root then f = root
+        end
+        f
+      end
+
+      def termdef_boilerplate_climb_up(clause, container)
+        container.at(".//*[@id = '#{clause['id']}']") or return clause
+        while (n = clause.parent)
+          n.at(".//definitions") and break
+          clause = n
+          n["id"] == container["id"] and break
+        end
+        clause
+      end
+
+      def termdef_boilerplate_insert1(sect, xmldoc, isodoc)
+        ins = sect.at("./title")
+        if (ins2 = sect.at("./clause[@type = 'boilerplate'] | " \
+                "./note[@type = 'boilerplate']"))
+          ins2.text.strip.downcase == "(default)" or return
+          ins2.children = " "
+          ins = ins2.children.first
+        end
+        term_defs_boilerplate(ins, xmldoc.xpath(".//termdocsource"),
+                              sect.at(".//term"), sect.at(".//p"), isodoc)
+      end
+    end
+  end
+end

lib/metanorma/standoc/cleanup_terms_designations.rb

@@ -156,8 +156,7 @@ def term_termsource_to_designation(xmldoc)
           while %w(domain subject).include? p&.name
             p = p.previous_element
           end
-          DESIGNATOR.include?(p&.name) or
-            next
+          DESIGNATOR.include?(p&.name) or next
           related2pref(p) << t.remove
         end
       end

lib/metanorma/standoc/cleanup_xref.rb

@@ -176,7 +176,7 @@ def xref_compound_cleanup(xmldoc)
 
       def xref_compound_cleanup1(xref, locations)
         xref.children.empty? and xref.children = "<sentinel/>"
-        xref_parse_compound_locations(locations).reverse.each do |y|
+        xref_parse_compound_locations(locations).reverse_each do |y|
           xref.children.first.previous =
             "<xref target='#{y[1]}' connective='#{y[0]}'/>"
         end

lib/metanorma/standoc/init.rb

@@ -103,6 +103,7 @@ def init_i18n(node)
       def init_biblio(node)
         @no_isobib_cache = node.attr("no-isobib-cache")
         @no_isobib = node.attr("no-isobib")
+        @flush_caches = node.attr("flush-caches")
         init_bib_log
         @bibdb = nil
         init_bib_caches(node)

lib/metanorma/standoc/ref_queue.rb

@@ -228,7 +228,7 @@ def init_iev_caches(node)
             @iev_globalname = global_ievcache_name
           @iev_localname = local_ievcache_name(node.attr("local-cache") ||
                                                node.attr("local-cache-only"))
-          if node.attr("flush-caches")
+          if @flush_caches
             FileUtils.rm_f @iev_globalname unless @iev_globalname.nil?
             FileUtils.rm_f @iev_localname unless @iev_localname.nil?
           end

lib/metanorma/standoc/term_lookup_cleanup.rb

@@ -4,7 +4,7 @@ module Metanorma
   module Standoc
     # Intelligent term lookup xml modifier
     class TermLookupCleanup
-      AUTO_GEN_ID_REGEXP = /\A_/.freeze
+      AUTO_GEN_ID_REGEXP = /\A_/
 
       attr_reader :xmldoc, :lookup, :log
 
@@ -130,7 +130,7 @@ def remove_missing_ref_msg(node, target, type)
       end
 
       def remove_missing_ref_msg1(_node, target, ret)
-        target2 = "_#{target.downcase.gsub('-', '_')}"
+        target2 = "_#{target.downcase.tr('-', '_')}"
         if @terms_tags[target] || @terms_tags[target2]
           ret.strip!
           ret += ". Did you mean to point to a subterm?"

lib/metanorma/standoc/utils.rb

@@ -100,8 +100,9 @@ def xml_encode(text)
       # wrapped in <sections>
       def adoc2xml(text, flavour)
         Nokogiri::XML(text).root and return text
+        f = @flush_caches ? ":flush-caches:\n" : ""
         c = Asciidoctor.convert("= X\nA\n:semantic-metadata-headless: true\n" \
-                                ":novalid:\n\n#{text}\n",
+                                ":no-isobib:\n#{f}:novalid:\n\n#{text}\n",
                                 backend: flavour, header_footer: true)
         Nokogiri::XML(c).at("//xmlns:sections")
       end