fix nim-lang#8871 runnableExamples now preserves source code comments…

…, litterals, and all formatting

compiler/ast.nim

@@ -1029,6 +1029,7 @@ const
   nkFloatLiterals* = {nkFloatLit..nkFloat128Lit}
   nkLambdaKinds* = {nkLambda, nkDo}
   declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef}
+  routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef}
   procDefs* = nkLambdaKinds + declarativeDefs
 
   nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}

compiler/docgen.nim

@@ -17,7 +17,7 @@ import
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, cgi, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees, tables, nimpaths
+  pathutils, trees, tables, nimpaths, renderverbatim
 
 const
   exportSection = skField
@@ -494,8 +494,8 @@ proc runAllExamples(d: PDoc) =
       rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
       # removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove
 
-proc prepareExample(d: PDoc; n: PNode): string =
-  ## returns `rdoccmd` for this runnableExamples
+proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
+  ## returns `rdoccmd` and source code for this runnableExamples
   var rdoccmd = ""
   if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid")
   if n.len == 3:
@@ -512,10 +512,11 @@ proc prepareExample(d: PDoc; n: PNode): string =
       docComment,
       newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
   runnableExamples.info = n.info
-
+  let ret = extractRunnableExamplesSource(d.conf, n)
   for a in n.lastSon: runnableExamples.add a
+  # we could also use `ret` instead here, to keep sources verbatim
   writeExample(d, runnableExamples, rdoccmd)
-  result = rdoccmd
+  result = (rdoccmd, ret)
   when false:
     proc extractImports(n: PNode; result: PNode) =
       if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
@@ -529,67 +530,110 @@ proc prepareExample(d: PDoc; n: PNode): string =
     for imp in imports: runnableExamples.add imp
     runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)
 
-proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previousIsRunnable: var bool) =
+proc renderNimCodeOld(d: PDoc, n: PNode, dest: var Rope) =
+  ## this is a rather hacky way to get rid of the initial indentation
+  ## that the renderer currently produces:
+  # deadcode
+  var i = 0
+  var body = n.lastSon
+  if body.len == 1 and body.kind == nkStmtList and
+      body.lastSon.kind == nkStmtList:
+    body = body.lastSon
+  for b in body:
+    if i > 0: dest.add "\n"
+    inc i
+    nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
+
+type RunnableState = enum
+  rsStart
+  rsComment
+  rsRunnable
+  rsDone
+
+proc getAllRunnableExamplesImpl(d: PDoc; n, orig: PNode; dest: var Rope, state: RunnableState): RunnableState =
   ##[
-  previousIsRunnable: keep track of whether previous sibling was a runnableExample (true if 1st sibling though).
-  This is to ensure this works:
+  Simple state machine to tell whether we render runnableExamples and doc comments.
+  This is to ensure that we can interleave runnableExamples and doc comments freely;
+  the logic is easy to change but currently a doc comment following another doc comment
+  will not render, to avoid rendering in following case:
+
   proc fn* =
     runnableExamples: discard
     ## d1
     runnableExamples: discard
     ## d2
 
-    ## d3 # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
-
-  It also works with:
-  proc fn* =
-    ## d0
-    runnableExamples: discard
-    ## d1
-
-    etc
+    ## internal explanation  # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
+    discard # some code
   ]##
-  # xxx: checkme: owner check instead? this fails with the $nim/nimdoc/tester.nim test
+
+  # xxx: orig is deadcode
+  # owner check instead? this fails with the $nim/nimdoc/tester.nim test
   # now that we're calling `genRecComment` only from here (to maintain correct order wrt runnableExample)
   # if n.info.fileIndex != orig.info.fileIndex: return
+
   case n.kind
   of nkCommentStmt:
-    if previousIsRunnable:
+    if state in {rsStart, rsRunnable}:
       dest.add genRecComment(d, n)
-    previousIsRunnable = false
+      return rsComment
   of nkCallKinds:
     if isRunnableExamples(n[0]) and
-        n.len >= 2 and n.lastSon.kind == nkStmtList:
-      previousIsRunnable = true
-      let rdoccmd = prepareExample(d, n)
+        n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}:
+      let (rdoccmd, code) = prepareExample(d, n)
       var msg = "Example:"
       if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
       dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
           "\n\\textbf{$1}\n", [msg.rope])
       inc d.listingCounter
       let id = $d.listingCounter
       dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
-      # this is a rather hacky way to get rid of the initial indentation
-      # that the renderer currently produces:
-      var i = 0
-      var body = n.lastSon
-      if body.len == 1 and body.kind == nkStmtList and
-          body.lastSon.kind == nkStmtList:
-        body = body.lastSon
-      for b in body:
-        if i > 0: dest.add "\n"
-        inc i
-        nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil)
+      when true:
+        var dest2 = ""
+        renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+        dest.add dest2
+      else:
+        renderNimCodeOld(d, n, dest)
       dest.add(d.config.getOrDefault"doc.listing_end" % id)
-  else: previousIsRunnable = false
-
-  var previousIsRunnable2 = true
-  for i in 0..<n.safeLen:
-    getAllRunnableExamplesRec(d, n[i], orig, dest, previousIsRunnable2)
-
-proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
-  var previousIsRunnable = true
-  getAllRunnableExamplesRec(d, n, n, dest, previousIsRunnable)
+      return rsRunnable
+  else: discard
+  return rsDone
+    # change this to `rsStart` if you want to keep generating doc comments
+    # and runnableExamples that occur after some code in routine
+
+proc getRoutineBody(n: PNode): PNode =
+  ## works around a transf bug D20200526T163511 where `result` messes AST
+  # TODO: try w templates and macro, see if they're affected
+  result = n[^1]
+  case result.kind
+  of nkSym:
+    result = n[^2]
+    case result.kind
+      of nkAsgn:
+        doAssert result[0].kind == nkSym
+        doAssert result.len == 2
+        result = result[1]
+      else: # eg: nkStmtList
+        discard
+  else:
+    discard
+
+proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) =
+  var n = n
+  let orig = n
+  var state = rsStart
+  template fn(n2) =
+    state = getAllRunnableExamplesImpl(d, n2, orig, dest, state)
+  case n.kind
+  of routineDefs:
+    n = n.getRoutineBody
+    case n.kind
+    of nkCommentStmt, nkCallKinds: fn(n)
+    else:
+      for i in 0..<n.safeLen:
+        fn(n[i])
+        if state == rsDone: return
+  else: fn(n)
 
 proc isVisible(d: PDoc; n: PNode): bool =
   result = false
@@ -765,12 +809,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   var literal, plainName = ""
   var kind = tkEof
   var comm: Rope = nil
-  # skipping this (and doing it inside getAllRunnableExamples) would fix order in
-  # case of a runnableExample appearing before a doccomment, but would cause other
-  # issues
-  comm.add genRecComment(d, n)
-  if n.kind in declarativeDefs:
+  if n.kind in routineDefs:
     getAllRunnableExamples(d, n, comm)
+  else:
+    comm.add genRecComment(d, n)
+
   var r: TSrcGen
   # Obtain the plain rendered string for hyperlink titles.
   initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,

compiler/msgs.nim

@@ -435,18 +435,21 @@ proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
 proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
   conf.m.fileInfos[fileIdx.int32].lines.add line
 
-proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
-  if i.fileIndex.int32 < 0: return ""
-
-  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
+  result = conf.m.fileInfos[fileIdx.int32].lines.len
+  if result == 0:
     try:
-      for line in lines(toFullPathConsiderDirty(conf, i)):
-        addSourceLine conf, i.fileIndex, line.string
+      for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
+        addSourceLine conf, fileIdx, line.string
     except IOError:
       discard
-  assert i.fileIndex.int32 < conf.m.fileInfos.len
+    result = conf.m.fileInfos[fileIdx.int32].lines.len
+
+proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
+  if i.fileIndex.int32 < 0: return ""
+  let num = numLines(conf, i.fileIndex)
   # can happen if the error points to EOF:
-  if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return ""
+  if i.line.int > num: return ""
 
   result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]
 

compiler/renderverbatim.nim

@@ -0,0 +1,65 @@
+import strutils
+from xmltree import addEscaped
+
+import ast, options, msgs
+import packages/docutils/highlite
+
+proc lastNodeRec(n: PNode): PNode =
+  result = n
+  while result.safeLen > 0: result = result[^1]
+
+proc isInIndentationBlock(src: string, indent: int): bool =
+  #[
+  we stop at the first de-indentation; there's an inherent ambiguity with non
+  doc comments since they can have arbitrary indentation, so we just take the
+  practical route and require a runnableExamples to keep its code (including non
+  doc comments) to its indentation level.
+  ]#
+  for j in 0..<indent:
+    if src.len <= j: return true
+    if src[j] == ' ': continue
+    return false
+  return true
+
+proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
+  ## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
+  ## we'd need to check performance impact to enable it for nimdoc.
+  let first = n.lastSon.info
+  let last = n.lastNodeRec.info
+  var ret = ""
+  var info = first
+  var indent = info.col
+  let numLines = numLines(conf, info.fileIndex).uint16
+  var lastNonemptyPos = 0
+  for line in first.line..<numLines:
+    info.line = line
+    let src = sourceLine(conf, info)
+    if line > last.line and not isInIndentationBlock(src, indent):
+      break
+    if line > first.line: ret.add "\n"
+    if src.len > indent:
+      ret.add src[indent..^1]
+      lastNonemptyPos = ret.len
+  ret = ret[0..<lastNonemptyPos]
+  return ret
+
+proc renderNimCode*(result: var string, code: string, isLatex = false) =
+  var toknizr: GeneralTokenizer
+  initGeneralTokenizer(toknizr, code)
+  var buf = ""
+  template append(kind, val) =
+    buf.setLen 0
+    buf.addEscaped(val)
+    let class = tokenClassToStr[kind]
+    if isLatex:
+      result.addf "\\span$1{$2}" % [class, buf]
+    else:
+      result.addf  "<span class=\"$1\">$2</span>" % [class, buf]
+
+  while true:
+    getNextToken(toknizr, langNim)
+    case toknizr.kind
+    of gtEof: break  # End Of File (or string)
+    else:
+      # TODO: avoid alloc; maybe toOpenArray
+      append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))

nimdoc/testproject/expected/testproject.html

@@ -173,7 +173,9 @@ <h1 class="title">testproject</h1>
 <li>
   <a class="reference reference-toplevel" href="#18" id="68">Templates</a>
   <ul class="simple simple-toc-section">
-      <li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
+      <li><a class="reference" href="#myfn.t"
+    title="myfn()"><wbr />myfn<span class="attachedType"></span></a></li>
+  <li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
     title="foo(a, b: SomeType)"><wbr />foo<span class="attachedType"></span></a></li>
 
   </ul>
@@ -187,13 +189,11 @@ <h1 class="title">testproject</h1>
 
   <p class="module-desc">This is the top level module.
 <p><strong class="examples_text">Example:</strong></p>
-<pre class="listing"><span class="Keyword">import</span>
-  <span class="Identifier">subdir</span> <span class="Operator">/</span> <span class="Identifier">subdir_b</span> <span class="Operator">/</span> <span class="Identifier">utils</span>
-
-<span class="Identifier">doAssert</span> <span class="Identifier">bar</span><span class="Other">(</span><span class="DecNumber">3</span><span class="Other">,</span> <span class="DecNumber">4</span><span class="Other">)</span> <span class="Operator">==</span> <span class="DecNumber">7</span>
-<span class="Identifier">foo</span><span class="Other">(</span><span class="Identifier">enumValueA</span><span class="Other">,</span> <span class="Identifier">enumValueB</span><span class="Other">)</span>
-<span class="Keyword">for</span> <span class="Identifier">x</span> <span class="Keyword">in</span> <span class="StringLit">&quot;xx&quot;</span><span class="Other">:</span>
-  <span class="Keyword">discard</span></pre></p>
+<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">subdir</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">subdir_b</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">utils</span><span class="Whitespace">
+</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="Identifier">bar</span><span class="Punctuation">(</span><span class="DecNumber">3</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="DecNumber">4</span><span class="Punctuation">)</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">7</span><span class="Whitespace">
+</span><span class="Identifier">foo</span><span class="Punctuation">(</span><span class="Identifier">enumValueA</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="Identifier">enumValueB</span><span class="Punctuation">)</span><span class="Whitespace">
+</span><span class="Comment"># bug #11078</span><span class="Whitespace">
+</span><span class="Keyword">for</span><span class="Whitespace"> </span><span class="Identifier">x</span><span class="Whitespace"> </span><span class="Keyword">in</span><span class="Whitespace"> </span><span class="StringLit">&quot;xx&quot;</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">discard</span></pre></p>
   <div class="section" id="6">
 <h1><a class="toc-backref" href="#6">Imports</a></h1>
 <dl class="item">
@@ -331,6 +331,26 @@ <h1><a class="toc-backref" href="#17">Macros</a></h1>
 <div class="section" id="18">
 <h1><a class="toc-backref" href="#18">Templates</a></h1>
 <dl class="item">
+<a id="myfn.t"></a>
+<dt><pre><span class="Keyword">template</span> <a href="#myfn.t"><span class="Identifier">myfn</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
+<dd>
+
+
+<p><strong class="examples_text">Example:</strong></p>
+<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">std</span><span class="Operator">/</span><span class="Identifier">strutils</span><span class="Whitespace">
+</span><span class="Comment">## line doc comment</span><span class="Whitespace">
+</span><span class="Comment"># bar</span><span class="Whitespace">
+</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace">
+</span><span class="LongComment">##[
+foo
+bar
+]##</span><span class="Whitespace">
+</span><span class="Identifier">doAssert</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">not</span><span class="Whitespace"> </span><span class="StringLit">&quot;foo&quot;</span><span class="Operator">.</span><span class="Identifier">startsWith</span><span class="Whitespace"> </span><span class="StringLit">&quot;ba&quot;</span><span class="Whitespace">
+</span><span class="Keyword">block</span><span class="Punctuation">:</span><span class="Whitespace">
+  </span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="HexNumber">0xff</span><span class="Whitespace"> </span><span class="Comment"># elu par cette crapule</span><span class="Whitespace">
+</span><span class="Comment"># should be in</span></pre>
+
+</dd>
 <a id="foo.t,SomeType,SomeType"></a>
 <dt><pre><span class="Keyword">template</span> <a href="#foo.t%2CSomeType%2CSomeType"><span class="Identifier">foo</span></a><span class="Other">(</span><span class="Identifier">a</span><span class="Other">,</span> <span class="Identifier">b</span><span class="Other">:</span> <a href="subdir/subdir_b/utils.html#SomeType"><span class="Identifier">SomeType</span></a><span class="Other">)</span></pre></dt>
 <dd>

nimdoc/testproject/expected/theindex.html

@@ -143,6 +143,10 @@ <h1 class="title">Index</h1>
 <li><a class="reference external"
           data-doc-search-tag="testproject: isValid[T](x: T): bool" href="testproject.html#isValid%2CT">testproject: isValid[T](x: T): bool</a></li>
           </ul></dd>
+<dt><a name="myfn" href="#myfn"><span>myfn:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: myfn()" href="testproject.html#myfn.t">testproject: myfn()</a></li>
+          </ul></dd>
 <dt><a name="someFunc" href="#someFunc"><span>someFunc:</span></a></dt><dd><ul class="simple">
 <li><a class="reference external"
           data-doc-search-tag="testproject: someFunc()" href="testproject.html#someFunc">testproject: someFunc()</a></li>

nimdoc/testproject/testproject.nim

@@ -9,6 +9,22 @@ runnableExamples:
   # bug #11078
   for x in "xx": discard
 
+template myfn*() =
+  runnableExamples:
+    import std/strutils
+    ## line doc comment
+    # bar
+    doAssert "'foo" == "'foo"
+    ##[
+    foo
+    bar
+    ]##
+    doAssert: not "foo".startsWith "ba"
+    block:
+      discard 0xff # elu par cette crapule
+    # should be in
+  # should be out
+
 const
   C_A* = 0x7FF0000000000000'f64
   C_B* = 0o377'i8