From d182094b0cef14f7af82a636d92426ae8bd3169b Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Sat, 29 Feb 2020 16:30:31 -0800 Subject: [PATCH] fix #8871 runnableExamples now preserves source code comments, litterals, and all formatting --- compiler/docgen.nim | 40 +++++++----- compiler/msgs.nim | 19 +++--- compiler/renderverbatim.nim | 65 ++++++++++++++++++++ nimdoc/testproject/expected/testproject.html | 36 ++++++++--- nimdoc/testproject/expected/theindex.html | 4 ++ nimdoc/testproject/testproject.nim | 16 +++++ 6 files changed, 147 insertions(+), 33 deletions(-) create mode 100644 compiler/renderverbatim.nim diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 2072945973ce7..ddfbeb549847a 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -17,7 +17,7 @@ import packages/docutils/rst, packages/docutils/rstgen, json, xmltree, cgi, trees, types, typesrenderer, astalgo, lineinfos, intsets, - pathutils, trees, tables, nimpaths + pathutils, trees, tables, nimpaths, renderverbatim const exportSection = skField @@ -494,8 +494,8 @@ proc runAllExamples(d: PDoc) = rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string]) # removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove -proc prepareExample(d: PDoc; n: PNode): string = - ## returns `rdoccmd` for this runnableExamples +proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] = + ## returns `rdoccmd` and source code for this runnableExamples var rdoccmd = "" if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid") if n.len == 3: @@ -512,10 +512,11 @@ proc prepareExample(d: PDoc; n: PNode): string = docComment, newTree(nkImportStmt, newStrNode(nkStrLit, d.filename))) runnableExamples.info = n.info - + let ret = extractRunnableExamplesSource(d.conf, n) for a in n.lastSon: runnableExamples.add a + # we could also use `ret` instead here, to keep sources verbatim writeExample(d, runnableExamples, rdoccmd) - result = rdoccmd + result = (rdoccmd, ret) when false: proc extractImports(n: PNode; result: PNode) = if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}: @@ -561,7 +562,7 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previous if isRunnableExamples(n[0]) and n.len >= 2 and n.lastSon.kind == nkStmtList: previousIsRunnable = true - let rdoccmd = prepareExample(d, n) + let (rdoccmd, code) = prepareExample(d, n) var msg = "Example:" if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd dispA(d.conf, dest, "\n

$1

\n", @@ -569,17 +570,22 @@ proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope, previous inc d.listingCounter let id = $d.listingCounter dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"]) - # this is a rather hacky way to get rid of the initial indentation - # that the renderer currently produces: - var i = 0 - var body = n.lastSon - if body.len == 1 and body.kind == nkStmtList and - body.lastSon.kind == nkStmtList: - body = body.lastSon - for b in body: - if i > 0: dest.add "\n" - inc i - nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil) + when true: + var dest2 = "" + renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex) + dest.add dest2 + else: + # this is a rather hacky way to get rid of the initial indentation + # that the renderer currently produces: + var i = 0 + var body = n.lastSon + if body.len == 1 and body.kind == nkStmtList and + body.lastSon.kind == nkStmtList: + body = body.lastSon + for b in body: + if i > 0: dest.add "\n" + inc i + nodeToHighlightedHtml(d, b, dest, {renderRunnableExamples}, nil) dest.add(d.config.getOrDefault"doc.listing_end" % id) else: previousIsRunnable = false diff --git a/compiler/msgs.nim b/compiler/msgs.nim index a26aa0a3cef67..00efff3aab0a6 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -435,18 +435,21 @@ proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool = proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) = conf.m.fileInfos[fileIdx.int32].lines.add line -proc sourceLine*(conf: ConfigRef; i: TLineInfo): string = - if i.fileIndex.int32 < 0: return "" - - if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0: +proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int = + result = conf.m.fileInfos[fileIdx.int32].lines.len + if result == 0: try: - for line in lines(toFullPathConsiderDirty(conf, i)): - addSourceLine conf, i.fileIndex, line.string + for line in lines(toFullPathConsiderDirty(conf, fileIdx).string): + addSourceLine conf, fileIdx, line.string except IOError: discard - assert i.fileIndex.int32 < conf.m.fileInfos.len + result = conf.m.fileInfos[fileIdx.int32].lines.len + +proc sourceLine*(conf: ConfigRef; i: TLineInfo): string = + if i.fileIndex.int32 < 0: return "" + let num = numLines(conf, i.fileIndex) # can happen if the error points to EOF: - if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return "" + if i.line.int > num: return "" result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1] diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim new file mode 100644 index 0000000000000..9513353cd92ac --- /dev/null +++ b/compiler/renderverbatim.nim @@ -0,0 +1,65 @@ +import strutils +from xmltree import addEscaped + +import ast, options, msgs +import packages/docutils/highlite + +proc lastNodeRec(n: PNode): PNode = + result = n + while result.safeLen > 0: result = result[^1] + +proc isInIndentationBlock(src: string, indent: int): bool = + #[ + we stop at the first de-indentation; there's an inherent ambiguity with non + doc comments since they can have arbitrary indentation, so we just take the + practical route and require a runnableExamples to keep its code (including non + doc comments) to its indentation level. + ]# + for j in 0.. last.line and not isInIndentationBlock(src, indent): + break + if line > first.line: ret.add "\n" + if src.len > indent: + ret.add src[indent..^1] + lastNonemptyPos = ret.len + ret = ret[0..$2" % [class, buf] + + while true: + getNextToken(toknizr, langNim) + case toknizr.kind + of gtEof: break # End Of File (or string) + else: + # TODO: avoid alloc; maybe toOpenArray + append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1)) diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index aa29c0c0dcbd4..257cb581b06b7 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -173,7 +173,9 @@

testproject

  • Templates @@ -187,13 +189,11 @@

    testproject

    This is the top level module.

    Example:

    -
    import
-  subdir / subdir_b / utils
-
-doAssert bar(3, 4) == 7
-foo(enumValueA, enumValueB)
-for x in "xx":
-  discard

    +
    import subdir / subdir_b / utils
+doAssert bar(3, 4) == 7
+foo(enumValueA, enumValueB)
+# bug #11078
+for x in "xx": discard

    Imports

    @@ -331,6 +331,26 @@

    Macros

    Templates

    + +
    template myfn()
    +
    + + +

    Example:

    +
    import std/strutils
+## line doc comment
+# bar
+doAssert "'foo" == "'foo"
+##[
+foo
+bar
+]##
+doAssert: not "foo".startsWith "ba"
+block:
+  discard 0xff # elu par cette crapule
+# should be in
    + +
    template foo(a, b: SomeType)
    diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index a76f5961f6b88..f78be11eeef16 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -143,6 +143,10 @@

    Index

  • testproject: isValid[T](x: T): bool
    • +
    myfn:
    someFunc:
    • testproject: someFunc()
      • diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim index d1fcf58cd83e4..b2976fe9a5495 100644 --- a/nimdoc/testproject/testproject.nim +++ b/nimdoc/testproject/testproject.nim @@ -9,6 +9,22 @@ runnableExamples: # bug #11078 for x in "xx": discard +template myfn*() = + runnableExamples: + import std/strutils + ## line doc comment + # bar + doAssert "'foo" == "'foo" + ##[ + foo + bar + ]## + doAssert: not "foo".startsWith "ba" + block: + discard 0xff # elu par cette crapule + # should be in + # should be out + const C_A* = 0x7FF0000000000000'f64 C_B* = 0o377'i8