Skip to content

Commit

Permalink
fix nim-lang#8871 runnableExamples now preserves source code comments…
Browse files Browse the repository at this point in the history
…, litterals, and all formatting
  • Loading branch information
timotheecour committed May 24, 2020
1 parent 0cc22de commit 1c769d4
Show file tree
Hide file tree
Showing 5 changed files with 119 additions and 97 deletions.
93 changes: 4 additions & 89 deletions compiler/docgen.nim
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ import
packages/docutils/rst, packages/docutils/rstgen,
json, xmltree, cgi, trees, types,
typesrenderer, astalgo, lineinfos, intsets,
pathutils, trees, tables
pathutils, trees, tables,
renderverbatim

const
exportSection = skField
Expand Down Expand Up @@ -492,91 +493,6 @@ proc runAllExamples(d: PDoc) =
rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
# removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove

proc lastNodeRec(n: PNode): PNode =
# MOVE
result = n
while result.safeLen > 0:
result = result[^1]

proc isInIndentationBlock(src: string, indent: int): bool =
#[
MOVE
returns whether source code line `src` is indented within indentation block
indented at `indent`, taking care of comments,
Limitation: following extreme edge case isn't supported, it's trivial
to work around it. As a benefit, we dont miss comments after the last
declaration inside runnableExamples, which alternative implementations would.
TODO: handle nested comments properly
runnableExamples:
echo [
1,
# this comment works fine
2,
#[
sneaky_indentation # this comment won't work
]#
] # this will be wrongly skipped because of previous edge case
]#

for j in 0..<indent:
# if src.len <= j or src[j] == '#': return true
if src.len <= j: return true
if src[j] == ' ': continue
return false
return true

proc extractRunnableExamplesSource(d: PDoc; n: PNode): string =
## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
## we'd need to check performance impact to enable it for nimdoc.
let first = n.lastSon.info
let last = n.lastNodeRec.info
var ret = ""
var info = first
var indent = info.col
let numLines = numLines(d.conf, info.fileIndex).uint16
var lastNonemptyPos = 0
for line in first.line..<numLines:
info.line = line
let src = sourceLine(d.conf, info)
if line > last.line and not isInIndentationBlock(src, indent):
# if line > last.line:
break
if line > first.line: ret.add "\n"
if src.len > indent:
ret.add src[indent..^1]
lastNonemptyPos = ret.len
ret = ret[0..<lastNonemptyPos]
return ret


when true:
import packages/docutils/highlite
import strutils
from xmltree import addEscaped

proc renderHtml(result: var string, code: string) =
var toknizr: GeneralTokenizer
initGeneralTokenizer(toknizr, code)
var buf = ""
template append(kind, val) =
buf.setLen 0
buf.addEscaped(val)
let class = tokenClassToStr[kind]
# TODO: use dispA
# dispA(d.conf, result, "<span class=\"StringLit\">$1</span>", "\\spanStringLit{$1}", [escLit])
# result.add "<span class=\"$1\">$2</span>" % [class, val]
result.add "<span class=\"$1\">$2</span>" % [class, buf]

while true:
getNextToken(toknizr, langNim)
case toknizr.kind
of gtEof: break # End Of File (or string)
else:
# TODO: avoid alloc; maybe toOpenArray
append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))

proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
## returns `rdoccmd` and source code for this runnableExamples
var rdoccmd = ""
Expand All @@ -594,10 +510,9 @@ proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
docComment,
newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
runnableExamples.info = n.info
let ret = extractRunnableExamplesSource(d, n)
echo ret
let ret = extractRunnableExamplesSource(d.conf, n)
for a in n.lastSon: runnableExamples.add a
# TODO: use ret instead here too?
# we could also use `ret` instead here, to keep sources verbatim
writeExample(d, runnableExamples, rdoccmd)
result = (rdoccmd, ret)
when false:
Expand Down
67 changes: 67 additions & 0 deletions compiler/renderverbatim.nim
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
import strutils
from xmltree import addEscaped

import ast, options, msgs
import packages/docutils/highlite

proc lastNodeRec(n: PNode): PNode =
# MOVE
result = n
while result.safeLen > 0:
result = result[^1]

proc isInIndentationBlock(src: string, indent: int): bool =
#[
we stop at the first de-indentation; there's an inherent ambiguity with non
doc comments since they can have arbitrary indentation, so we just take the
practical route and require a runnableExamples to keep its code (including non
doc comments) to its indentation level.
]#
for j in 0..<indent:
if src.len <= j: return true
if src[j] == ' ': continue
return false
return true

proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
## we'd need to check performance impact to enable it for nimdoc.
let first = n.lastSon.info
let last = n.lastNodeRec.info
var ret = ""
var info = first
var indent = info.col
let numLines = numLines(conf, info.fileIndex).uint16
var lastNonemptyPos = 0
for line in first.line..<numLines:
info.line = line
let src = sourceLine(conf, info)
if line > last.line and not isInIndentationBlock(src, indent):
break
if line > first.line: ret.add "\n"
if src.len > indent:
ret.add src[indent..^1]
lastNonemptyPos = ret.len
ret = ret[0..<lastNonemptyPos]
return ret

proc renderHtml*(result: var string, code: string) =
var toknizr: GeneralTokenizer
initGeneralTokenizer(toknizr, code)
var buf = ""
template append(kind, val) =
buf.setLen 0
buf.addEscaped(val)
let class = tokenClassToStr[kind]
# TODO: use dispA
# dispA(d.conf, result, "<span class=\"StringLit\">$1</span>", "\\spanStringLit{$1}", [escLit])
# result.add "<span class=\"$1\">$2</span>" % [class, val]
result.add "<span class=\"$1\">$2</span>" % [class, buf]

while true:
getNextToken(toknizr, langNim)
case toknizr.kind
of gtEof: break # End Of File (or string)
else:
# TODO: avoid alloc; maybe toOpenArray
append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))
36 changes: 28 additions & 8 deletions nimdoc/testproject/expected/testproject.html
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,9 @@ <h1 class="title">testproject</h1>
<li>
<a class="reference reference-toplevel" href="#18" id="68">Templates</a>
<ul class="simple simple-toc-section">
<li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
<li><a class="reference" href="#myfn.t"
title="myfn()"><wbr />myfn<span class="attachedType"></span></a></li>
<li><a class="reference" href="#foo.t%2CSomeType%2CSomeType"
title="foo(a, b: SomeType)"><wbr />foo<span class="attachedType"></span></a></li>

</ul>
Expand All @@ -184,13 +186,11 @@ <h1 class="title">testproject</h1>

<p class="module-desc">This is the top level module.
<p><strong class="examples_text">Example:</strong></p>
<pre class="listing"><span class="Keyword">import</span>
<span class="Identifier">subdir</span> <span class="Operator">/</span> <span class="Identifier">subdir_b</span> <span class="Operator">/</span> <span class="Identifier">utils</span>

<span class="Identifier">doAssert</span> <span class="Identifier">bar</span><span class="Other">(</span><span class="DecNumber">3</span><span class="Other">,</span> <span class="DecNumber">4</span><span class="Other">)</span> <span class="Operator">==</span> <span class="DecNumber">7</span>
<span class="Identifier">foo</span><span class="Other">(</span><span class="Identifier">enumValueA</span><span class="Other">,</span> <span class="Identifier">enumValueB</span><span class="Other">)</span>
<span class="Keyword">for</span> <span class="Identifier">x</span> <span class="Keyword">in</span> <span class="StringLit">&quot;xx&quot;</span><span class="Other">:</span>
<span class="Keyword">discard</span></pre></p>
<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">subdir</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">subdir_b</span><span class="Whitespace"> </span><span class="Operator">/</span><span class="Whitespace"> </span><span class="Identifier">utils</span><span class="Whitespace">
</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="Identifier">bar</span><span class="Punctuation">(</span><span class="DecNumber">3</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="DecNumber">4</span><span class="Punctuation">)</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="DecNumber">7</span><span class="Whitespace">
</span><span class="Identifier">foo</span><span class="Punctuation">(</span><span class="Identifier">enumValueA</span><span class="Punctuation">,</span><span class="Whitespace"> </span><span class="Identifier">enumValueB</span><span class="Punctuation">)</span><span class="Whitespace">
</span><span class="Comment"># bug #11078</span><span class="Whitespace">
</span><span class="Keyword">for</span><span class="Whitespace"> </span><span class="Identifier">x</span><span class="Whitespace"> </span><span class="Keyword">in</span><span class="Whitespace"> </span><span class="StringLit">&quot;xx&quot;</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">discard</span></pre></p>
<div class="section" id="6">
<h1><a class="toc-backref" href="#6">Imports</a></h1>
<dl class="item">
Expand Down Expand Up @@ -328,6 +328,26 @@ <h1><a class="toc-backref" href="#17">Macros</a></h1>
<div class="section" id="18">
<h1><a class="toc-backref" href="#18">Templates</a></h1>
<dl class="item">
<a id="myfn.t"></a>
<dt><pre><span class="Keyword">template</span> <a href="#myfn.t"><span class="Identifier">myfn</span></a><span class="Other">(</span><span class="Other">)</span></pre></dt>
<dd>


<p><strong class="examples_text">Example:</strong></p>
<pre class="listing"><span class="Keyword">import</span><span class="Whitespace"> </span><span class="Identifier">std</span><span class="Operator">/</span><span class="Identifier">strutils</span><span class="Whitespace">
</span><span class="Comment">## line doc comment</span><span class="Whitespace">
</span><span class="Comment"># bar</span><span class="Whitespace">
</span><span class="Identifier">doAssert</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace"> </span><span class="Operator">==</span><span class="Whitespace"> </span><span class="StringLit">&quot;&apos;foo&quot;</span><span class="Whitespace">
</span><span class="LongComment">##[
foo
bar
]##</span><span class="Whitespace">
</span><span class="Identifier">doAssert</span><span class="Punctuation">:</span><span class="Whitespace"> </span><span class="Keyword">not</span><span class="Whitespace"> </span><span class="StringLit">&quot;foo&quot;</span><span class="Operator">.</span><span class="Identifier">startsWith</span><span class="Whitespace"> </span><span class="StringLit">&quot;ba&quot;</span><span class="Whitespace">
</span><span class="Keyword">block</span><span class="Punctuation">:</span><span class="Whitespace">
</span><span class="Keyword">discard</span><span class="Whitespace"> </span><span class="HexNumber">0xff</span><span class="Whitespace"> </span><span class="Comment"># elu par cette crapule</span><span class="Whitespace">
</span><span class="Comment"># should be in</span></pre>

</dd>
<a id="foo.t,SomeType,SomeType"></a>
<dt><pre><span class="Keyword">template</span> <a href="#foo.t%2CSomeType%2CSomeType"><span class="Identifier">foo</span></a><span class="Other">(</span><span class="Identifier">a</span><span class="Other">,</span> <span class="Identifier">b</span><span class="Other">:</span> <a href="subdir/subdir_b/utils.html#SomeType"><span class="Identifier">SomeType</span></a><span class="Other">)</span></pre></dt>
<dd>
Expand Down
4 changes: 4 additions & 0 deletions nimdoc/testproject/expected/theindex.html
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ <h1 class="title">Index</h1>
<li><a class="reference external"
data-doc-search-tag="testproject: isValid[T](x: T): bool" href="testproject.html#isValid%2CT">testproject: isValid[T](x: T): bool</a></li>
</ul></dd>
<dt><a name="myfn" href="#myfn"><span>myfn:</span></a></dt><dd><ul class="simple">
<li><a class="reference external"
data-doc-search-tag="testproject: myfn()" href="testproject.html#myfn.t">testproject: myfn()</a></li>
</ul></dd>
<dt><a name="someFunc" href="#someFunc"><span>someFunc:</span></a></dt><dd><ul class="simple">
<li><a class="reference external"
data-doc-search-tag="testproject: someFunc()" href="testproject.html#someFunc">testproject: someFunc()</a></li>
Expand Down
16 changes: 16 additions & 0 deletions nimdoc/testproject/testproject.nim
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,22 @@ runnableExamples:
# bug #11078
for x in "xx": discard

template myfn*() =
runnableExamples:
import std/strutils
## line doc comment
# bar
doAssert "'foo" == "'foo"
##[
foo
bar
]##
doAssert: not "foo".startsWith "ba"
block:
discard 0xff # elu par cette crapule
# should be in
# should be out

const
C_A* = 0x7FF0000000000000'f64
C_B* = 0o377'i8
Expand Down

0 comments on commit 1c769d4

Please sign in to comment.