diff --git a/compiler/front/cli_reporter.nim b/compiler/front/cli_reporter.nim index 081aff61c88..c14ff403ea6 100644 --- a/compiler/front/cli_reporter.nim +++ b/compiler/front/cli_reporter.nim @@ -143,7 +143,7 @@ proc toStr(conf: ConfigRef, loc: TLineInfo, dropExt: bool = false): string = ## Convert location to printable string conf.wrap( "$1($2, $3)" % [ - toFilenameOption(conf, loc.fileIndex, conf.filenameOption).dropExt(dropExt), + conf.toMsgFilename(loc.fileIndex).dropExt(dropExt), $loc.line, $(loc.col + ColOffset) ], @@ -3083,7 +3083,12 @@ proc reportBody*(conf: ConfigRef, r: ExternalReport): string = result = "$1 is not a valid number" % r.cmdlineProvided of rextInvalidValue: - result = r.cmdlineError + result = ("Unexpected value for " & + "the $1. Expected one of $2, but got '$3'") % [ + r.cmdlineSwitch, + r.cmdlineAllowed.mapIt("'" & it & "'").join(", "), + r.cmdlineProvided + ] of rextUnexpectedValue: result = "Unexpected value for $1. Expected one of $2" % [ diff --git a/compiler/front/commands.nim b/compiler/front/commands.nim index a259b7e49c2..cd48b05cda9 100644 --- a/compiler/front/commands.nim +++ b/compiler/front/commands.nim @@ -45,7 +45,9 @@ import front/[ condsyms, options, - msgs + msgs, + cli_reporter, + sexp_reporter ], backend/[ extccomp @@ -1121,6 +1123,17 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; else: conf.localReport(info, invalidSwitchValue @["abs", "canonical", "legacyRelProj"]) + of "msgformat": + case arg.normalize: + of "text": + conf.setReportHook cli_reporter.reportHook + + of "sexp": + conf.setReportHook sexp_reporter.reportHook + + else: + conf.localReport(info, invalidSwitchValue @["text", "sexp"]) + of "processing": incl(conf, cnCurrent, rsemProcessing) incl(conf, cnMainPackage, rsemProcessing) @@ -1269,6 +1282,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "nilseqs", "nilchecks", "mainmodule", "m", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch) + else: if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg) else: invalidCmdLineOption(conf, pass, switch, info) diff --git a/compiler/front/options.nim b/compiler/front/options.nim index 6d7b293fd36..99bcfa9e1a9 100644 --- a/compiler/front/options.nim +++ b/compiler/front/options.nim @@ -421,7 +421,6 @@ type ) {.closure.} ## All ## textual output from the compiler goes through this callback. writeHook*: proc(conf: ConfigRef, output: string, flags: MsgFlags) {.closure.} - structuredReportHook*: ReportHook cppCustomNamespace*: string vmProfileData*: ProfileData diff --git a/compiler/front/sexp_reporter.nim b/compiler/front/sexp_reporter.nim new file mode 100644 index 00000000000..451477717a4 --- /dev/null +++ b/compiler/front/sexp_reporter.nim @@ -0,0 +1,185 @@ +## Implementation of the structured CLI message generator. Using +## `--msgFormat=sexp` will make compiler switch to the report hook +## implemented in this module. + +import + experimental/[ + sexp, + diff, + colortext, + sexp_diff + ], + ast/[ + lineinfos, + ast, + reports + ], + front/[ + options, + msgs + ], + std/[ + strutils + ] + + +var writeConf: ConfigRef + +import std/options as std_options + +proc addFields[T](s: var SexpNode, r: T, ignore: seq[string] = @[]) + +proc sexp[T: object | tuple](obj: T): SexpNode = + result = newSList() + addFields(result, obj) + +proc sexp[T: object | tuple](obj: ref T): SexpNode = + result = newSList() + addFields(result, obj[]) + +proc sexp*[E: enum](e: E): SexpNode = newSSymbol($e) + +proc sexpItems*[T](s: T): SexpNode = + result = newSList() + for item in items(s): + result.add sexp(item) + + +proc sexp*[T](s: seq[T]): SexpNode = sexpItems(s) +proc sexp*[R, T](s: array[R, T]): SexpNode = sexpItems(s) +proc sexp*[I](s: set[I]): SexpNode = sexpItems(s) +proc sexp*(s: cstring): SexpNode = sexp($s) + +proc sexp*(v: SomeInteger): SexpNode = newSInt(BiggestInt(v)) +proc sexp*(id: FileIndex): SexpNode = + sexp(writeConf.toMsgFilename(id)) + + +iterator sexpFields[T](obj: T, ignore: seq[string] = @[]): SexpNode = + for name, value in fieldPairs(obj): + var pass = true + when value is ref or value is ptr: + if isNil(value): + pass = false + + when value is seq or value is string: + if len(value) == 0: + pass = false + + when value is TLineInfo: + if pass and value == unknownLineInfo: + pass = false + + when value is ReportLineInfo: + if pass and value.isValid(): + pass = false + + if pass and name in ignore: + pass = false + + if pass: + yield newSKeyword(name, sexp(value)) + + +proc add*(other: var SexpNode, str: string, expr: SexpNode) = + other.add newSSymbol(":" & str) + other.add expr + +proc sexp*[T](o: Option[T]): SexpNode = + if o.isNone: newSNil() else: sexp(o.get()) + +proc addFields[T](s: var SexpNode, r: T, ignore: seq[string] = @[]) = + for item in sexpFields(r, ignore): + s.add item + +proc sexp*(i: ReportLineInfo): SexpNode = + convertSexp([ + writeConf.formatPath(i.file).sexp(), + sexp(i.line), + sexp(i.col) + ]) + +proc sexp*(i: TLineInfo): SexpNode = + convertSexp([sexp(i.fileIndex), sexp(i.line), sexp(i.col)]) + +proc sexp*(e: StackTraceEntry): SexpNode = + result = newSList() + result.addFields(e, @["filename"]) + result.add newSKeyword( + "filename", writeConf.formatPath($e.filename).sexp()) + + +proc sexp*(typ: PType): SexpNode = + if typ.isNil: return newSNil() + result = newSList() + result.add newSSymbol(($typ.kind)[2 ..^ 1]) + if typ.sons.len > 0: + result.add("sons", sexp(typ.sons)) + +proc sexp*(node: PNode): SexpNode = + if node.isNil: return newSNil() + + result = newSList() + result.add newSSymbol(($node.kind)[2 ..^ 1]) + case node.kind: + of nkCharLit..nkUInt64Lit: result.add sexp(node.intVal) + of nkFloatLit..nkFloat128Lit: result.add sexp(node.floatVal) + of nkStrLit..nkTripleStrLit: result.add sexp(node.strVal) + of nkSym: result.add newSSymbol(node.sym.name.s) + of nkIdent: result.add newSSymbol(node.ident.s) + else: + for node in node.sons: + result.add sexp(node) + +proc sexp*(t: PSym): SexpNode = + convertSexp([ + substr($t.kind, 2).newSSymbol(), + name = sexp(t.name.s), + info = sexp(t.info) + ]) + + +proc reportHook*(conf: ConfigRef, r: Report): TErrorHandling = + writeConf = conf + let wkind = conf.writabilityKind(r) + + if wkind == writeDisabled: + return + + else: + var s = newSList() + s.add newSSymbol(multiReplace($r.kind, { + "rsem": "Sem", + "rpar": "Par", + "rlex": "Lex", + "rint": "Int", + "rext": "Ext", + "rdbg": "Dbg", + "rback": "Bck", + })) + s.add newSSymbol(":severity") + s.add sexp(conf.severity(r)) + + let f = @["kind"] + + case r.category: + of repLexer: s.addFields(r.lexReport, f) + of repParser: s.addFields(r.parserReport, f) + of repCmd: s.addFields(r.cmdReport, f) + of repSem: + if r.kind == rsemProcessingStmt: + s.addFields(r.semReport, f & "node") + + else: + s.addFields(r.semReport, f) + + of repDebug: s.addFields(r.debugReport) + of repInternal: s.addFields(r.internalReport) + of repBackend: s.addFields(r.backendReport) + of repExternal: s.addFields(r.externalReport) + + if wkind == writeForceEnabled: + echo s.toLine().toString(conf.useColor) + + else: + conf.writeln(s.toLine().toString(conf.useColor)) diff --git a/compiler/sem/semcall.nim b/compiler/sem/semcall.nim index d3dfdaa65c8..c409e7e1253 100644 --- a/compiler/sem/semcall.nim +++ b/compiler/sem/semcall.nim @@ -113,7 +113,7 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode, firstMismatch: z.firstMismatch, diagnostics: z.diagnostics, isDiagnostic: z.diagnosticsEnabled or efExplain in flags - )) + )) else: # Symbol table has been modified. Restart and pre-calculate all syms diff --git a/compiler/vm/vm.nim b/compiler/vm/vm.nim index 2ed0b8f7c67..c5b8b56f2b0 100644 --- a/compiler/vm/vm.nim +++ b/compiler/vm/vm.nim @@ -17,7 +17,6 @@ import parseutils ], ast/[ - astalgo, lineinfos, renderer, # toStrLit implementation trees, diff --git a/doc/testament.rst b/doc/testament.rst index 2af80e06968..5b3e4bd0a40 100644 --- a/doc/testament.rst +++ b/doc/testament.rst @@ -91,87 +91,45 @@ you have to run at least 1 test *before* generating a report: Writing Tests ============= -Example "template" **to edit** and write a Testament unit: -.. code-block:: nim +``description`` - textual description of the test. **Highly** recomended to +add one - in the future testament might use this to provide better teardown +reports, or notify about ``knownIssue`` state transitions. - discard """ +Test execution options +---------------------- - # What actions to expect completion on. - # Options: - # "compile": expect successful compilation - # "run": expect successful compilation and execution - # "reject": expect failed compilation. The "reject" action can catch - # {.error.} pragmas but not {.fatal.} pragmas because - # {.fatal.} pragmas guarantee that compilation will be aborted. - action: "run" - - # The exit code that the test is expected to return. Typically, the default - # value of 0 is fine. Note that if the test will be run by valgrind, then - # the test will exit with either a code of 0 on success or 1 on failure. - exitcode: 0 - - # Provide an `output` string to assert that the test prints to standard out - # exactly the expected string. Provide an `outputsub` string to assert that - # the string given here is a substring of the standard out output of the - # test. - output: "" - outputsub: "" - - # Whether to sort the output lines before comparing them to the desired - # output. - sortoutput: true - - # Each line in the string given here appears in the same order in the - # compiler output, but there may be more lines that appear before, after, or - # in between them. - nimout: ''' - a very long, - multi-line - string''' +- ``action`` - What actions to expect completion on. + - ``"compile"``: expect successful compilation + - ``"run"``: expect successful compilation and execution + - ``"reject"``: expect failed compilation. The "reject" action can catch + `{.error.}` pragmas but not `{.fatal.}` pragmas because `{.fatal.}` pragmas + guarantee that compilation will be aborted. - # This is the Standard Input the test should take, if any. - input: "" +- ``batchable``: Can be run in batch mode, or not. - # Error message the test should print, if any. - errormsg: "" +- ``joinable``: Can be run Joined with other tests to run all togheter, or + not. Defaults to `true` - # Can be run in batch mode, or not. - batchable: true +- ``timeout`` Timeout seconds to run the test. Fractional values are supported. - # Can be run Joined with other tests to run all togheter, or not. - joinable: true +- ``cmd``: Command the test should use to run. If left out or an empty + string is provided, the command is taken to be: ``"nim $target --hints:on + -d:testing --nimblePath:build/deps/pkgs $options $file"`` You can use the + ``$target``, ``$options``, and ``$file`` placeholders in your own + command, too. - # On Linux 64-bit machines, whether to use Valgrind to check for bad memory - # accesses or memory leaks. On other architectures, the test will be run - # as-is, without Valgrind. - # Options: - # true: run the test with Valgrind - # false: run the without Valgrind - # "leaks": run the test with Valgrind, but do not check for memory leaks - valgrind: false # Can use Valgrind to check for memory leaks, or not (Linux 64Bit only). + example: ``"nim c -r $file"`` - # Command the test should use to run. If left out or an empty string is - # provided, the command is taken to be: - # "nim $target --hints:on -d:testing --nimblePath:build/deps/pkgs $options $file" - # You can use the $target, $options, and $file placeholders in your own - # command, too. - cmd: "nim c -r $file" +- ``targets`` Targets to run the test into (c, cpp, objc, js). - # Maximum generated temporary intermediate code file size for the test. - maxcodesize: 666 +- ``matrix`` flags with which to run the test, delimited by `;` - # Timeout seconds to run the test. Fractional values are supported. - timeout: 1.5 +- ``disabled`` Conditions that will skip this test. Use of multiple + "disabled" clauses is permitted. - # Targets to run the test into (c, cpp, objc, js). - targets: "c js" + .. code-block:: nim - # flags with which to run the test, delimited by `;` - matrix: "; -d:release; -d:caseFoo -d:release" - - # Conditions that will skip this test. Use of multiple "disabled" clauses - # is permitted. disabled: "bsd" # Can disable OSes... disabled: "win" disabled: "32bit" # ...or architectures @@ -179,10 +137,74 @@ Example "template" **to edit** and write a Testament unit: disabled: "azure" # ...or pipeline runners disabled: true # ...or can disable the test entirely - """ - assert true - assert 42 == 42, "Assert error message" +- ``knownIssue`` + +Compiler output assertions +-------------------------- + +- ``errormsg``: Error message the test should print, if any. + + +- ``nimout`` Each line in the string given here appears in the same order + in the compiler output, but there may be more lines that appear before, + after, or in between them. Note that specifying multiline strings for + testament spec inside of the `discard """` section requires using + triple single quotes `'` + + .. code-block:: nim + + nimout: ''' + a very long, + multi-line + string''' + +- ``nimoutFull``: true/false, controls whether full compiler output must be + asserted, or only presence of error messages + +- ``maxcodesize``: Max side of the resulting codegen file for a test + +In addition to ``nimout`` message annotations testament also allows to +supply hints, warnings and error messages directly in the source code using +specially formatted comments, starting with ``#[tt.``. For example, if you +want to assert that error message is genrated, you can write a following +test: + +.. code-block:: + + {.error: "Error message".} #[tt.Error + ^ "Error message" + ]# + +File, line and column information are automatically inferred from the +position of the ``^`` marker in the annotation body. + +Binary output assertions +------------------------ + +- ``exitcode``: The exit code that the test is expected to return. + Typically, the default value of 0 is fine. Note that if the test will be + run by valgrind, then the test will exit with either a code of 0 on + success or 1 on failure. + +- ``output``, ``outsub``: Provide an `output` string to assert that the + test prints to standard out exactly the expected string. Provide an + `outputsub` string to assert that the string given here is a substring of + the standard out output of the test. + +- ``sortoutput`` Whether to sort the output lines before comparing them to + the desired output. + +- ``input``: this is the Standard Input the test should take, if any. + + +- ``valgrind`` On Linux 64-bit machines, whether to use Valgrind to check + for bad memory accesses or memory leaks. On other architectures, the + test will be run as-is, without Valgrind. + + - ``true``: run the test with Valgrind + - ``false``: run the without Valgrind + - ``"leaks"``: run the test with Valgrind, but do not check for memory leaks * As you can see the "Spec" is just a `discard """ """`. * Spec has sane defaults, so you don't need to provide them all, any simple assert will work just fine. @@ -191,10 +213,129 @@ Example "template" **to edit** and write a Testament unit: * Has some built-in CI compatibility, like Azure Pipelines, etc. * `Testament supports inlined error messages on tests, basically comments with the expected error directly on the code. `_ +Reading test outputs +==================== + +Testament supports two different modes of interaction with the compiler - +structured and unstructured. Unstructured interaction mode (currently +default) allows user to specify exact compiler output that should be +produced by input file and then compares it based on the ``nimoutFull`` +configuration options. + +If tests are mismatched, failure message is generated, showing failed +diffs. + +.. code-block:: + + discard """ + nimout: ''' + Expected unstructured compiler output + ''' + """ + + static: + echo "Expected unstructured output" + +In that case comparison is performed between two regular string blocks. +Since each entry is not wide enough (not wider than current terminal) they +are printed side-by side to make it easier to spot the difference. +Mismatches are also highlighted in the terminal. + +.. code-block:: diff + + - Expected unstructured compiler output + Expected unstructured output + - ? + +.. note:: expected (on the left) outout has two lines deleted - trailing + ``'''`` in the testament spec is placed on the next line, so it + is considered to be a string literal of ``"Expected unstructured + compiler output\n"`` + + +Structured mismatches +--------------------- + +In structured output mode, compiler writes out S-expressions for each +output diagnostic entry, one per line. + +If testament is used in structured mode, all expected compiler reports - +both inline and written in ``nimout`` are collected in a single list that +is matched against produced output directly. Failure message shows best +possible mismatch annotations for the given output. For example, given test +like this, testament output will contain two mismatches for both failures. + +.. code-block:: nim + :linenos: + + discard """ + nimoutFormat: sexp + cmd: "nim c --msgFormat=sexp --skipUserCfg --hints=on --hint=all:off --hint=User:on --filenames:canonical $file" + nimout: ''' + (User :str "User Hint" :location ("tfile.nim" 8 _)) + ''' + """ + + {.hint: "User hint".} + + {.hint: "Another hint".} #[tt.Hint + ^ (User :str "Another hint") ]# + + +Both inline and ``nimout`` annotations are compared. Both have errors, so +the best possible mapping is presented as an error ('best' because it is +generally impossible to find correct place to insert inline annotation +somewhere in ``nimout``, without potentially messing up ordering. +Unstructured output simply sets inline annotations to a higher priority and +searches for them first) + +.. code-block:: nim + + Expected inline Hint annotation at tfile.nim(11, 7): + + - (User :location ("tfile.nim" 11 7) :severity Hint :str "Another hint") + + Given: + + + (User :location ("tfile.nim" 11 6) :severity Hint :str "Another hint") + + + :location[2] expected 7, but got 6 ([7->6]) + + Expected: + + - (User :location ("tfile.nim" 8 _) :str "User Hint") + + Given: + + + (User :location ("tfile.nim" 9 6) :severity Hint :str "User hint") + + + :str expected "User Hint", but got "User hint" ("User [Hint"->hint"]) + :location[1] expected 8, but got 9 ([8->9]) + +Compiler printed reports + +.. code-block:: nim + + (User :severity Hint :str "User hint" :location ("tfile.nim" 8 6)) + (User :severity Hint :str "Another hint" :location ("tfile.nim" 10 6)) + +And they were matched against full list of expected entries. For the first +entry there is a mismatch in ``:location[2]``, and for secdon one there is +a string value error (in ``:str``) and another mismatch in location data. + +To make it easier to spot differences between string values the inline diff +is added for the message. Test Examples ============== +Structured test examples +------------------------ + +Unstructured old, style test examples +-------------------------------------- + Expected to fail: .. code-block:: nim diff --git a/doc/tools.rst b/doc/tools.rst index 0de4ac91432..5b31860ecb3 100644 --- a/doc/tools.rst +++ b/doc/tools.rst @@ -20,9 +20,6 @@ The standard distribution ships with the following tools: and obtain useful information like the definition of symbols or suggestions for completion. -- | `C2nim `_ - | C to Nim source converter. Translates C header files to Nim. - - | `niminst `_ | niminst is a tool to generate an installer for a Nim program. diff --git a/lib/experimental/colordiff.nim b/lib/experimental/colordiff.nim new file mode 100644 index 00000000000..7fa04673463 --- /dev/null +++ b/lib/experimental/colordiff.nim @@ -0,0 +1,709 @@ +## This module implements formatting logic for colored text diffs - both +## multiline and inline. +## +## All formatting is generated in colored text format and can be later +## formatted in both plaintext and formatted manners using +## `colortext.toString` + +import ./diff, ./colortext +import std/[sequtils, strutils, strformat, algorithm] + +export toString, `$`, myersDiff, shiftDiffed + +proc colorDollar*[T](arg: T): ColText = toColText($arg) + +type + DiffFormatConf* = object + ## Diff formatting configuration + maxUnchanged*: int ## Max number of the unchanged lines after which + ## they will be no longer show. Can be used to compact large diffs with + ## small mismatched parts. + maxUnchangedWords*: int ## Max number of the unchanged words in a + ## single line. Can be used to compact long lines with small mismatches + showLines*: bool ## Show line numbers in the generated diffs + lineSplit*: proc(str: string): seq[string] ## Split line + ## into chunks for formatting + sideBySide*: bool ## Show line diff with side-by-side (aka github + ## 'split' view) or on top of each other (aka 'unified') + explainInvisible*: bool ## If diff contains invisible characters - + ## trailing whitespaces, control characters, escapes and ANSI SGR + ## formatting - show them directly. + inlineDiffSeparator*: ColText ## Text to separate words in the inline split + formatChunk*: proc( + text: string, + mode, secondary: SeqEditKind, + inline: bool + ): ColText ## Format + ## mismatched text. `mode` is the mismatch kind, `secondary` is used + ## for `sekChanged` to annotated which part was deleted and which part + ## was added. + groupLine*: bool ## For multiline edit operations - group consecutive + ## Edit operations into single chunks. + groupInline*: bool ## For inline edit operations - group consecutive + ## edit operations into single chunks. + explainChar*: proc(ch: char): string ## Convert invisible character + ## (whitespace or control) to human-readable representation - + +func unified*(conf: DiffFormatConf): bool = + ## Check if config is used to build unified diff + not conf.sideBySide + +proc chunk( + conf: DiffFormatConf, text: string, + mode: SeqEditKind, secondary: SeqEditKind = mode, + inline: bool = false + ): ColText = + ## Format text mismatch chunk using `formatChunk` callback + conf.formatChunk(text, mode, secondary, inline) + +func splitKeepSeparator*(str: string, sep: set[char] = {' '}): seq[string] = + ## Default implementaion of the line splitter - split on `sep` characters + ## but don't discard them - they will be present in the resulting output. + var prev = 0 + var curr = 0 + while curr < str.len: + if str[curr] in sep: + if prev != curr: + result.add str[prev ..< curr] + + prev = curr + while curr < str.high and str[curr + 1] == str[curr]: + inc curr + + result.add str[prev .. curr] + inc curr + prev = curr + + else: + inc curr + + if prev < curr: + result.add str[prev ..< curr] + +proc formatDiffed*[T]( + ops: seq[SeqEdit], + oldSeq, newSeq: seq[T], + conf: DiffFormatConf + ): tuple[oldLine, newLine: ColText] = + ## Generate colored formatting fothe levenshtein edit operation using + ## format configuration. Return old formatted line and new formatted line. + + var unchanged = 0 + var oldLine: seq[ColText] + var newLine: seq[ColText] + for idx, op in ops: + case op.kind: + of sekKeep: + if unchanged < conf.maxUnchanged: + oldLine.add conf.chunk(oldSeq[op.sourcePos], sekKeep) + newLine.add conf.chunk(newSeq[op.targetPos], sekKeep) + inc unchanged + + of sekDelete: + oldLine.add conf.chunk(oldSeq[op.sourcePos], sekDelete) + unchanged = 0 + + of sekInsert: + newLine.add conf.chunk(newSeq[op.targetPos], sekInsert) + unchanged = 0 + + of sekReplace: + oldLine.add conf.chunk(oldSeq[op.sourcePos], sekReplace, sekDelete) + newLine.add conf.chunk(newSeq[op.targetPos], sekReplace, sekInsert) + unchanged = 0 + + of sekNone: + assert false + + of sekTranspose: + discard + + return ( + oldLine.join(conf.inlineDiffSeparator), + newLine.join(conf.inlineDiffSeparator) + ) + + + + +func visibleName(ch: char): tuple[unicode, ascii: string] = + ## Get visible name of the character. + case ch: + of '\x00': ("␀", "[NUL]") + of '\x01': ("␁", "[SOH]") + of '\x02': ("␂", "[STX]") + of '\x03': ("␃", "[ETX]") + of '\x04': ("␄", "[EOT]") + of '\x05': ("␅", "[ENQ]") + of '\x06': ("␆", "[ACK]") + of '\x07': ("␇", "[BEL]") + of '\x08': ("␈", "[BS]") + of '\x09': ("␉", "[HT]") + of '\x0A': ("␤", "[LF]") + of '\x0B': ("␋", "[VT]") + of '\x0C': ("␌", "[FF]") + of '\x0D': ("␍", "[CR]") + of '\x0E': ("␎", "[SO]") + of '\x0F': ("␏", "[SI]") + of '\x10': ("␐", "[DLE]") + of '\x11': ("␑", "[DC1]") + of '\x12': ("␒", "[DC2]") + of '\x13': ("␓", "[DC3]") + of '\x14': ("␔", "[DC4]") + of '\x15': ("␕", "[NAK]") + of '\x16': ("␖", "[SYN]") + of '\x17': ("␗", "[ETB]") + of '\x18': ("␘", "[CAN]") + of '\x19': ("␙", "[EM]") + of '\x1A': ("␚", "[SUB]") + of '\x1B': ("␛", "[ESC]") + of '\x1C': ("␜", "[FS]") + of '\x1D': ("␝", "[GS]") + of '\x1E': ("␞", "[RS]") + of '\x1F': ("␟", "[US]") + of '\x7f': ("␡", "[DEL]") + of ' ': ("␣", "[SPC]") # Space + else: ($ch, $ch) + +proc toVisibleNames(conf: DiffFormatConf, str: string): string = + ## Convert all characters in the string into visible ones + for ch in str: + result.add conf.explainChar(ch) + + +proc toVisibleNames(conf: DiffFormatConf, split: seq[string]): seq[string] = + ## Convert all charactersw in all strings into visible ones. + if 0 < split.len(): + for part in split: + result.add conf.toVisibleNames(part) + +const Invis = { '\x00' .. '\x1F', '\x7F' } + +func scanInvisible(text: string, invisSet: var set[char]): bool = + ## Scan string for invisible characters from right to left, updating + ## active invisible set as needed. + var chIdx = text.high + while 0 <= chIdx: + if text[chIdx] in invisSet: + return true + + else: + invisSet = Invis + + dec chIdx + +func hasInvisible*(text: string, startSet: set[char] = Invis + {' '}): bool = + ## Does string have significant invisible characters? + var invisSet = startSet + if scanInvisible(text, invisSet): + return true + +func hasInvisible*(text: seq[string]): bool = + ## Do any of strings in text have signfificant invisible characters. + var idx = text.high + var invisSet = Invis + {' '} + while 0 <= idx: + # Iterate over all items from righ to left - until we found first + # visible character space is also considered significant, but removed + # afterwards, so `" a"/"a"` is not considered to have invisible + # characters. + if scanInvisible(text[idx], invisSet): + return true + dec idx + + +func hasInvisibleChanges(diff: seq[SeqEdit], oldSeq, newSeq: seq[string]): bool = + ## Is any change in the edit sequence invisible? + var start = Invis + {' '} + + proc invis(text: string): bool = + result = scanInvisible(text, start) + + # Iterate over all edits from right to left, updating active set of + # invisible characters as we got. + var idx = diff.high + while 0 <= idx: + let edit = diff[idx] + case edit.kind: + of sekDelete: + if oldSeq[edit.sourcePos].invis(): + return true + + of sekInsert: + if newSeq[edit.targetPos].invis(): + return true + + of sekNone, sekTranspose: + discard + + of sekKeep: + # Check for kept characters - this will update 'invisible' set if + # any found, so edits like `" X" -> "X"` are not considered as 'has + # invisible' + if oldSeq[edit.sourcePos].invis(): + discard + + of sekReplace: + if oldSeq[edit.sourcePos].invis() or + newSeq[edit.targetPos].invis(): + return true + + dec idx + +func diffFormatter*(useUnicode: bool = true): DiffFormatConf = + ## Default implementation of the diff formatter + ## + ## - split lines by whitespace + ## - no hidden lines or workds + ## - deleted: red, inserted: green, changed: yellow + ## - explain invisible differences with unicode + DiffFormatConf( + # Don't hide inline edit lines + maxUnchanged: high(int), + # Group edit operations for inline diff by default + groupInline: true, + # Show differences if there are any invisible characters + explainInvisible: true, + # Don't hide inline edit words + maxUnchangedWords: high(int), + showLines: false, + explainChar: ( + proc(ch: char): string = + let (uc, ascii) = visibleName(ch) + if useUnicode: uc else: ascii + ), + lineSplit: ( + # Split by whitespace + proc(a: string): seq[string] = splitKeepSeparator(a) + ), + sideBySide: false, + formatChunk: ( + proc(word: string, mode, secondary: SeqEditKind, inline: bool): ColText = + case mode: + of sekDelete: word + fgRed + of sekInsert: word + fgGreen + of sekKeep: word + fgDefault + of sekNone: word + fgDefault + of sekReplace, sekTranspose: + if inline and secondary == sekDelete: + "[" & (word + fgYellow) & " -> " + + elif inline and secondary == sekInsert: + (word + fgYellow) & "]" + + else: + word + fgYellow + ) + ) + +proc formatLineDiff*( + old, new: string, conf: DiffFormatConf, + ): tuple[oldLine, newLine: ColText] = + ## Format single line diff into old/new line edits. Optionally explain + ## all differences using options from `conf` + + let + oldSplit = conf.lineSplit(old) + newSplit = conf.lineSplit(new) + diffed = levenshteinDistance[string](oldSplit, newSplit) + + var oldLine, newLine: ColText + + if conf.explainInvisible and ( + diffed.operations.hasInvisibleChanges(oldSplit, newSplit) or + oldSplit.hasInvisible() or + newSplit.hasInvisible() + ): + (oldLine, newLine) = formatDiffed( + diffed.operations, + conf.toVisibleNames(oldSplit), + conf.toVisibleNames(newSplit), + conf + ) + + else: + (oldLine, newLine) = formatDiffed( + diffed.operations, + oldSplit, newSplit, + conf + ) + + return (oldLine, newLine) + + +template groupByIt[T](sequence: seq[T], op: untyped): seq[seq[T]] = + ## Group input sequence by value of the `op` into smaller subsequences + var res: seq[seq[T]] + var i = 0 + for item in sequence: + if i == 0: + res.add @[item] + + else: + if ((block: + let it {.inject.} = res[^1][0]; op)) == + ((block: + let it {.inject.} = item; op)): + res[^1].add item + + else: + res.add @[item] + + inc i + + res + +proc formatInlineDiff*( + src, target: seq[string], + diffed: seq[SeqEdit], + conf: DiffFormatConf + ): ColText = + ## Format inline edit operations for `src` and `target` sequences using + ## list of sequence edit operations `diffed`, formatting the result using + ## `conf` formatter. Consecutive edit operations are grouped together if + ## `conf.groupInline` is set to `true` + + var start = Invis + {' '} + var chunks: seq[ColText] + proc push( + text: string, + mode: SeqEditKind, + secondary: SeqEditKind = mode, + toLast: bool = false, + inline: bool = false + ) = + ## Push single piece of changed text to the resulting chunk sequence + ## after scanning for invisible characters. if `toLast` then add + ## directly to the last chunk - needed to avoid intermixing edit + ## visuals for the `sekReplace` edits which are the most important of + ## them all + var chunk: ColText + if conf.explainInvisible and scanInvisible(text, start): + chunk = conf.chunk( + conf.toVisibleNames(text), mode, secondary, inline = inline) + + else: + chunk = conf.chunk( + text, mode, secondary, inline = inline) + + if toLast: + chunks[^1].add chunk + + else: + chunks.add chunk + + let groups = + if conf.groupInline: + # Group edit operations by chunk - `[ins], [ins], [ins] -> [ins, ins, ins]` + # + # This is not specifically important for insertions and deletions, + # but pretty much mandatory for the 'change' operation, if we don't + # want to end up with the `h->He->El->Lo->O` instead of + # `hello->HELLO` + groupByIt(diffed, it.kind) + + else: + # Treat each group as a single edit operation if needed + mapIt(diffed, @[it]) + + var gIdx = groups.high + while 0 <= gIdx: + case groups[gIdx][0].kind: + of sekKeep: + var buf: string + for op in groups[gIdx]: + buf.add src[op.sourcePos] + + push(buf, sekKeep) + + of sekNone, sekTranspose: + discard + + of sekInsert: + var buf: string + for op in groups[gIdx]: + buf.add target[op.targetPos] + + push(buf, sekInsert) + + of sekDelete: + var buf: string + for op in groups[gIdx]: + buf.add src[op.sourcePos] + + push(buf, sekDelete) + + of sekReplace: + var sourceBuf, targetBuf: string + for op in groups[gIdx]: + sourceBuf.add src[op.sourcePos] + targetBuf.add target[op.targetPos] + + push(sourceBuf, sekReplace, sekDelete, inline = true) + # Force add directly to the last chunk + push(targetBuf, sekReplace, sekInsert, toLast = true, inline = true) + + dec gIdx + + # Because we iterated from right to left, all edit operations are placed + # in reverse as well, so this needs to be fixed + return chunks.reversed().join(conf.inlineDiffSeparator) + + +proc formatInlineDiff*( + src, target: string, conf: DiffFormatConf + ): ColText = + ## Generate inline string editing annotation for the source and target + ## string. Use `conf` for message mismatch configuration. + let + src = conf.lineSplit(src) + target = conf.lineSplit(target) + + return formatInlineDiff( + src, target, levenshteinDistance[string](src, target).operations, conf) + +type + BufItem = tuple[text: ColText, changed: bool, kind: SeqEditKind] ## Final + ## information about formatting line, only contains minimally necessary + ## data to do the final join formatting. + + +proc joinBuffer(oldText, newText: seq[BufItem], conf: DiffFormatConf): ColText = + ## Join two groups of formatted lines into final messages, using + ## parameters specified in the `conf` + coloredResult() + var first = true + proc addl() = + if not first: + add "\n" + first = false + + if conf.groupLine: + # Grouping line edits is not possible in the side-by-side + # representation, so going directly for a + var + lhsBuf: seq[BufItem] = @[oldText[0]] + rhsBuf: seq[BufItem] = @[newText[0]] + + proc addBuffers() = + for line in lhsBuf: + if line.changed: + addl() + add line.text + + for line in rhsBuf: + if line.changed: + add "\n" + add line.text + + + for (lhs, rhs) in zip(oldText[1..^1], newText[1..^1]): + if lhs.kind != lhsBuf[^1].kind or rhs.kind != rhsBuf[^1].kind: + # If one of the edit streaks finished, dump both results to output + # and continue + # + # - removed + added - removed + # - removed + added - removed + # - removed ? + added + # ~ kept ~ kept + added + # ~ kept ~ kept - removed + # ~ kept + # ~ kept + addBuffers() + lhsBuf = @[lhs] + rhsBuf = @[rhs] + + else: + lhsBuf.add lhs + rhsBuf.add rhs + + addBuffers() + + else: + var lhsMax = 0 + if conf.sideBySide: + for item in oldText: + lhsMax = max(item.text.len, lhsMax) + + var first = true + for (lhs, rhs) in zip(oldText, newText): + addl() + if conf.sideBySide: + add alignLeft(lhs.text, lhsMax + 3) + add rhs.text + + else: + add lhs.text + if rhs.changed: + add "\n" + add rhs.text + + +proc formatDiffed*( + shifted: ShiftedDiff, + oldSeq, newSeq: openArray[string], + conf: DiffFormatConf = diffFormatter() + ): ColText = + ## Format shifted multiline diff + ## + ## `oldSeq`, `newSeq` - sequence of lines (no newlines in strings + ## assumed) that will be formatted. + + + var + oldText, newText: seq[BufItem] + + # Max line number len for left and right side + let maxLhsIdx = len($shifted.oldShifted[^1].item) + let maxRhsIdx = len($shifted.newShifted[^1].item) + + proc editFmt(edit: SeqEditKind, idx: int, isLhs: bool): ColText = + ## Format prefix for edit operation for line at index `idx` + let editOps = [ + sekNone: "?", + sekKeep: "~", + sekInsert: "+", + sekReplace: "-+", + sekDelete: "-", + sekTranspose: "^v" + ] + + var change: string + # Make a `"+ "` or other prefix + if edit == sekNone and not isLhs: + # no trailing newlines for the filler lines on the rhs + change = editOps[edit] + + else: + change = alignLeft(editOps[edit], 2) + + # Optionally add line annotation + if conf.showLines: + if edit == sekNone: + change.add align(" ", maxLhsIdx) + + elif isLhs: + change.add align($idx, maxLhsIdx) + + else: + change.add align($idx, maxRhsIdx) + + # Wrap change chunk via provided callback and return the prefix + if edit == sekReplace: + return conf.chunk(change, edit, if isLhs: sekDelete else: sekInsert) + + else: + return conf.chunk(change, edit) + + # Number of unchanged lines + var unchanged = 0 + for (lhs, rhs, lhsDefault, rhsDefault, idx) in zipToMax( + shifted.oldShifted, shifted.newShifted + ): + if lhs.kind == sekKeep and rhs.kind == sekKeep: + if unchanged < conf.maxUnchanged: + inc unchanged + + else: + continue + + else: + unchanged = 0 + # Start new entry on the old line + oldText.add(( + editFmt(lhs.kind, lhs.item, true), + # Only case where lhs can have unchanged lines is for unified + # diff+filler + conf.unified and lhs.kind notin {sekNone}, + lhs.kind + )) + + # New entry on the new line + newText.add(( + editFmt(rhs.kind, rhs.item, false), + # Only newly inserted lines need to be formatted for the unified + # diff, everything else is displayed on the 'original' version. + conf.unified and rhs.kind in {sekInsert}, + rhs.kind + )) + + # Determine whether any of the lines is empty (old/new has len == 0) + var lhsEmpty, rhsEmpty: bool + if not lhsDefault and + not rhsDefault and + lhs.kind == sekDelete and + rhs.kind == sekInsert: + # Old part is deleted, new is inserted directly in place, show the + # line diff between those two (more detailed highlight of the + # modified parts in each version) + + let (oldLine, newLine) = formatLineDiff( + oldSeq[lhs.item], + newSeq[rhs.item], + conf + ) + + oldText[^1].text.add oldLine + newText[^1].text.add newLine + + + elif rhs.kind == sekInsert: + # Insert new and wrap in formatter + let tmp = newSeq[rhs.item] + rhsEmpty = tmp.len == 0 + # Append to the trailing new line + newText[^1].text.add conf.chunk(tmp, sekInsert) + + elif lhs.kind == sekDelete: + # Same as above, but for deletion and old text + let tmp = oldSeq[lhs.item] + lhsEmpty = tmp.len == 0 + oldText[^1].text.add conf.chunk(tmp, sekDelete) + + else: + # Everything else is mapped directly to each other + let ltmp = oldSeq[lhs.item] + lhsEmpty = ltmp.len == 0 + oldText[^1].text.add conf.chunk(ltmp, lhs.kind) + + let rtmp = newSeq[rhs.item] + rhsEmpty = rtmp.len == 0 + newText[^1].text.add conf.chunk(rtmp, rhs.kind) + + + # If line is not traling filler (for example new version had a +10 + # lines at bottom, so old file had to be padded at bottom as well to + # align), add newline to the diff to emphathisze file changes. + if lhsEmpty and idx < shifted.oldShifted.high: + oldText[^1].text.add conf.chunk( + conf.toVisibleNames("\n"), sekDelete) + + if rhsEmpty and idx < shifted.newShifted.high: + newText[^1].text.add conf.chunk( + conf.toVisibleNames("\n"), sekInsert) + + + return joinBuffer(oldText, newText, conf) + +proc formatDiffed*[T]( + oldSeq, newSeq: openArray[T], + conf: DiffFormatConf, + eqCmp: proc(a, b: T): bool = (proc(a, b: T): bool = a == b), + strConv: proc(a: T): string = (proc(a: T): string = $a) + ): ColText = + + formatDiffed( + myersDiff(oldSeq, newSeq, eqCmp).shiftDiffed(oldSeq, newSeq), + mapIt(oldSeq, strConv($it)), + mapIt(newSeq, strConv(it)), + conf + ) + + +proc formatDiffed*( + text1, text2: string, + conf: DiffFormatConf = diffFormatter() + ): ColText = + ## Format diff of two text blocks via newline split and default + ## `formatDiffed` implementation + formatDiffed(text1.split("\n"), text2.split("\n"), conf) diff --git a/lib/experimental/colortext.nim b/lib/experimental/colortext.nim index dfcac0b7fdb..29e5e45ded9 100644 --- a/lib/experimental/colortext.nim +++ b/lib/experimental/colortext.nim @@ -434,6 +434,15 @@ proc indent*( inc idx +func join*(text: seq[ColText], sep: ColText): ColText = + var first = true + for item in text: + if not first: + result.add sep + first = false + + result.add item + func stripLines*( text: ColText, leading: bool = false, @@ -619,7 +628,6 @@ const scaleRed: uint8 = 6 * 6 scaleGreen: uint8 = 6 - func `$`*(colored: ColRune): string = ## Convert to string with ansi escape sequences. To disable coloring use ## `toString` procedure instead. @@ -712,6 +720,9 @@ template coloredResult*(indentationStep: int = 2): untyped = else: return + proc addf(format: string, args: varargs[ColText, toColText]) = + outPtr[].addf(format, args) + template add(arg: untyped): untyped {.used.} = outPtr[].add arg template add(arg1, arg2: untyped): untyped {.used.} = outPtr[].add(arg1) @@ -785,3 +796,34 @@ func grid*(text: ColText): ColRuneGrid = ## Convert colored text to grid for line in lines(text): result.add line + +func addf*( + text: var ColText, + formatstr: string, + colored: varargs[ColText, toColText] + ) = + ## Interpolate `formatstr` using values from `colored` and add results to + ## the `text`. + ## + ## Iterpolation syntax is identical to the `std/strutils.addf` except + ## `$name` is currently not supported, so only positional interpolation + ## is available. + for fr in addfFragments(formatstr): + case fr.kind: + of addfDollar: + text.add "$" + + of addfText: + text.add fr.text + + of addfVar, addfExpr: + assert false, "var/expr formatting is not supported for colored text yet" + + of addfPositional, addfIndexed, addfBackIndexed: + let idx = if fr.kind == addfBackIndexed: len(colored) - fr.idx else: fr.idx + assert (0 <= idx and idx < colored.len) + text.add colored[idx] + +func `%`*(format: string, interpolate: openArray[ColText]): ColText = + ## Shorthand for colored text interpolation + result.addf(format, interpolate) diff --git a/lib/experimental/diff.nim b/lib/experimental/diff.nim index a217c27fe3d..1ee4c5083e5 100644 --- a/lib/experimental/diff.nim +++ b/lib/experimental/diff.nim @@ -353,7 +353,7 @@ type targetPos*: int ## Position in the target sequence proc levenshteinDistance*[T]( - str1, str2: openarray[T] + str1, str2: openArray[T] ): tuple[distance: int, operations: seq[SeqEdit]] = ## Compute edit distance between two item sequences, return list of edit ## operations necessary to transform `str1` into `str2` @@ -442,7 +442,7 @@ type proc myersDiff*[T]( - aSeq, bSeq: openarray[T], itemCmp: proc(x, y: T): bool): seq[SeqEdit] = + aSeq, bSeq: openArray[T], itemCmp: proc(x, y: T): bool): seq[SeqEdit] = ## Generate series of sequence edit operations necessary to trasnform ## `aSeq` into `bSeq`. For item equality comparison use `itemCmp` ## @@ -492,7 +492,8 @@ proc myersDiff*[T]( front[k] = (x, history) proc shiftDiffed*[T]( - diff: seq[SeqEdit], oldSeq, newSeq: openarray[T]): ShiftedDiff = + diff: seq[SeqEdit], oldSeq, newSeq: openArray[T]): ShiftedDiff = + ## Align diff operations against each other, for further formatting. for line in items(diff): case line.kind: @@ -530,6 +531,30 @@ proc shiftDiffed*[T]( result.newShifted.add((sekKeep, line.targetPos)) +iterator zipToMax*[T](lhs, rhs: seq[T], fill: T = default(T)): + tuple[lhs, rhs: T, rhsDefault, lhsDefault: bool, idx: int] = + ## Iterate each argumen to the end, filling in missing values with `fill` + ## argument. This is an opposite of the std built-in `zip` which iterates + ## up until `min(lhs.len, rhs.len)`. + ## + ## This is an internal iteratr, exported to be shared with `colordiff` + ## implementaion, but if you are implementing your own diff formatter, + ## you might consider using it as well. + + var idx = 0 + while idx < max(lhs.len, rhs.len): + if idx < lhs.len and idx < rhs.len: + yield (lhs[idx], rhs[idx], false, false, idx) + + elif idx < lhs.len: + yield (lhs[idx], fill, false, true, idx) + + else: + yield (fill, rhs[idx], true, false, idx) + + inc idx + + proc formatDiffed*( shifted: ShiftedDiff, oldSeq, newSeq: seq[string], @@ -601,7 +626,7 @@ Generated diff formatting does not contain trailing newline # Iterate over shifted diff sequence, construct formatted list of lines # that will be joined to final output. - for (lhs, rhs) in zip(shifted.oldShifted, shifted.newShifted): + for (lhs, rhs, lhsDefault, rhsDefault, _) in zipToMax(shifted.oldShifted, shifted.newShifted): oldText.add((editFmt(lhs.kind, lhs.item, true), true)) newText.add(( @@ -611,7 +636,10 @@ Generated diff formatting does not contain trailing newline not sideBySide and rhs.kind in {sekInsert} )) - if lhs.kind == sekDelete and rhs.kind == sekInsert: + if not lhsDefault and + not rhsDefault and + lhs.kind == sekDelete and + rhs.kind == sekInsert: oldText[^1].text.add oldSeq[lhs.item] newText[^1].text.add newSeq[rhs.item] @@ -628,7 +656,7 @@ Generated diff formatting does not contain trailing newline lhsMax = max(oldText[^1].text.len, lhsMax) var first = true - for (lhs, rhs) in zip(oldtext, newtext): + for (lhs, rhs) in zip(oldText, newText): if not first: # Avoid trailing newline of the diff formatting. result.add "\n" @@ -645,7 +673,7 @@ Generated diff formatting does not contain trailing newline result.add rhs.text -proc myersDiff*[T](aSeq, bSeq: openarray[T]): seq[SeqEdit] = +proc myersDiff*[T](aSeq, bSeq: openArray[T]): seq[SeqEdit] = ## Diff overload without explicit comparator proc - use default `==` for ## two items. myersDiff(aSeq, bSeq, proc(a, b: T): bool = a == b) diff --git a/nimsuggest/sexp.nim b/lib/experimental/sexp.nim similarity index 97% rename from nimsuggest/sexp.nim rename to lib/experimental/sexp.nim index 557f00866aa..67d2229055f 100644 --- a/nimsuggest/sexp.nim +++ b/lib/experimental/sexp.nim @@ -10,7 +10,7 @@ ## **Note:** Import ``nimsuggest/sexp`` to use this module import - hashes, strutils, lexbase, streams, unicode, macros + std/[hashes, strutils, lexbase, streams, unicode, macros, algorithm] import std/private/decode_helpers @@ -508,6 +508,20 @@ proc add*(father, child: SexpNode) = assert father.kind == SList father.elems.add(child) +proc addField*(node: SexpNode, name: string, value: SexpNode) = + ## Add `:name value` keyword pair to the `node` + node.add(newSKeyword(name, value)) + +proc getField*( + node: SexpNode, name: string, default: SexpNode = nil + ): SexpNode = + ## Iterate over direct subnodes of `node`, searching for the SKeyword + ## with name set to `name`. If found return it's `.value`, otherwise + ## return `default` + for field in node.elems: + if field.kind == SKeyword and field.getKey() == name: + return field.value + # ------------- pretty printing ---------------------------------------------- proc indent(s: var string, i: int) = @@ -609,6 +623,7 @@ proc toPretty(result: var string, node: SexpNode, indent = 2, ml = true, true, newIndent(currIndent, indent, ml)) result.add(")") + proc pretty*(node: SexpNode, indent = 2): string = ## Converts `node` to its Sexp Representation, with indentation and ## on multiple lines. diff --git a/lib/experimental/sexp_diff.nim b/lib/experimental/sexp_diff.nim new file mode 100644 index 00000000000..b3e00b24b81 --- /dev/null +++ b/lib/experimental/sexp_diff.nim @@ -0,0 +1,406 @@ +import + ./sexp, + ./colortext, + ./colordiff, + std/[ + strformat, + sequtils, + strutils, + tables, + intsets, + options, + algorithm + ] + +type IdxCostMap* = Table[(int, int), int] + +proc randomKey[K, V](table: OrderedTable[K, V]): K = + for k, v in pairs(table): + return k + +proc stablematch*( + lhsLen, rhsLen: int, + weight: proc(a, b: int): int, + order: SortOrder = SortOrder.Ascending + ): tuple[ + lhsIgnore, rhsIgnore: seq[int], + map: seq[tuple[pair: (int, int), cost: int]] + ] = + ## Do a weighted matching of the items in lhs and rhs sequences using + ## weight function. Return most cost-effective matching elements. + ## + ## - `lhsLen` and `rhsLen` lists number of the elements in each input + ## sequence + ## - `weight` - comparison proc that returns match score between + ## two items as position `a` and `b`. + ## - `order` - comparison ordering. If it is `Ascending` higher matching + ## cost is consdered better and replaces previous mappings. If `Descending` + ## prefer lower matching cost instead + ## + ## For generating mapping of two sequences make `weight` function a + ## closure and let it retrieve values as needed. + + var canTry: OrderedTable[int, seq[int]] + var rmap: OrderedTable[int, (int,int)] + + for l in 0 ..< lhsLen: + canTry[l] = @[] + for r in 0 ..< rhsLen: + canTry[l].add r + + proc getCost(l, r: int, res: var IdxCostMap): int = + if (l, r) notin res: + res[(l, r)] = weight(l, r) + + res[(l, r)] + + var tmp: IdxCostMap + while 0 < len(canTry): + let l = canTry.randomKey() + let r = canTry[l].pop() + if r in rmap: + let (oldL, _) = rmap[r] + let tryCost = getCost(l, r, tmp) + let otherCost = getCost(oldL, r, tmp) + let better = + if order == Ascending: + otherCost < tryCost + else: + otherCost > tryCost + + if better: + rmap[r] = (l, r) + + else: + discard getCost(l, r, tmp) + rmap[r] = (l, r) + + if canTry[l].len() == 0: + canTry.del l + + var tmpMap: seq[((int, int), int)] = toSeq(pairs(tmp)) + sort( + tmpMap, + proc(a, b: ((int, int), int)): int = + if a[1] == b[1]: + cmp(a[0], b[0]) + elif order == Descending: + cmp(a[1], b[1]) + else: + -cmp(a[1], b[1]) + ) + + var seenLset: IntSet + var seenRset: IntSet + + for (key, val) in tmpMap: + if key[0] notin seenLset and key[1] notin seenRset: + result.map.add((key, val)) + seenLset.incl key[0] + seenRset.incl key[1] + + for idx in 0 ..< rhsLen: + if idx notin seenRset: + result.rhsIgnore.add idx + + for idx in 0 ..< lhsLen: + if idx notin seenLset: + result.lhsIgnore.add idx + +export `$`, toString + +type + SexpPathPartKind = enum + ## Kind of the s-expression mismatch path art + spIndex + spKey + + SexpPathPart = object + ## S-expression mismatch part kind + case kind*: SexpPathPartKind + of spIndex: + index*: int ## Mismatch at index + + of spKey: + key*: string ## Mismatch for a given `:key` + + SexpPath* = seq[SexpPathPart] + + SexpMismatchKind* = enum + ## Possible kinds of the mismatches + smMissingKey ## Input data has no `:key` that was present in expected + smDifferentLiteral ## Target has different literal values from the expected + smDifferentSymbol ## Target has different symbol at position + smArrayLen ## Mismatched array len + smKindMismatch ## Different kinds of nodes - expected string but found + ## int for example + + SexpMismatch* = object + ## Single S-expression mismatch + path*: SexpPath ## Full path for the mismatched + case kind*: SexpMismatchKind + of smMissingKey: + key*: string ## Key missing in the input data + + of smDifferentLiteral, smKindMismatch, smArrayLen, smDifferentSymbol: + expected*, found*: SexpNode ## 'expected X' but 'found Y' error messages + arraydiff*: tuple[target, input: seq[int]] ## For comparison of the + ## lists keys - indices of the non-field elements. + +func sdiffPart*(key: string): SexpPathPart = + ## Create single S-expression key path part + SexpPathPart(key: key, kind: spKey) + +func sdiffPart*(index: int): SexpPathPart = + ## Create single S-expression index part + SexpPathPart(index: index, kind: spIndex) + + +func mismatch*(path: SexpPath, key: string): SexpMismatch = + ## Create missing key mismatch + SexpMismatch(kind: smMissingKey, key: key, path: path) + +proc mismatch( + kind: SexpMismatchKind, path: SexpPath, + expected, found: SexpNode + ): SexpMismatch = + ## Create expected/found mismatch + + result = SexpMismatch(kind: kind, path: path) + result.expected = expected + result.found = found + + +proc diff*(target, input: SexpNode): seq[SexpMismatch] = + ##[ + +Recursively iterate over target and input trees, find all mismatches. + +Comparison rules: + +- `_` in expected matches to anything +- Excess fields in `input` are discarded +- Missing fields in `target` are treated as errors +- List with keys are compared in two passes - only `:key` to `:key` + between two lists - in unordered manner. Then all remaining elements + are processed in the order of their appearance. +- Literals and kinds are compared directly with `==` + + ]## + + proc aux( + target, input: SexpNode, + path: SexpPath, + mismatches: var seq[SexpMismatch] + ) = + + if target.kind == SSymbol and target.getSymbol() == "_": + # `_` matches against everything and does not produce diffs + return + + elif target.kind != input.kind: + mismatches.add mismatch(smKindMismatch, path, target, input) + + else: + case target.kind: + of SInt: + if target.getNum() != input.getNum(): + mismatches.add mismatch(smDifferentLiteral, path, target, input) + + of SFloat: + if target.getFNum() != input.getFNum(): + mismatches.add mismatch(smDifferentLiteral, path, target, input) + + of SString: + if target.getStr() != input.getStr(): + mismatches.add mismatch(smDifferentLiteral, path, target, input) + + of SSymbol: + if target.getSymbol() != input.getSymbol(): + mismatches.add mismatch(smDifferentSymbol, path, target, input) + + of SList: + var + inputKeys: Table[string, int] + inputNonKeys, targetNonKeys: seq[int] + + for idx, item in pairs(input): + if item.kind == SKeyword: + inputKeys[item.getKey()] = idx + + else: + inputNonKeys.add idx + + for idx, item in pairs(target): + if item.kind == SKeyword: + let key = item.getKey() + if key in inputKeys: + aux( + item, + input[inputKeys[key]], + path & sdiffPart(key), mismatches) + + else: + mismatches.add mismatch(path, key) + + else: + targetNonKeys.add idx + + if inputNonKeys.len != targetNonKeys.len: + var mis = mismatch(smArrayLen, path, target, input) + mis.arraydiff = (targetNonKeys, inputNonKeys) + mismatches.add mis + + for idx in 0 ..< min(inputNonKeys.len, targetNonKeys.len): + aux( + target[targetNonKeys[idx]], + input[inputNonKeys[idx]], + path & sdiffPart(inputNonKeys[idx]), + mismatches + ) + + of SCons: + aux(target.car, input.car, path & sdiffPart(0), mismatches) + aux(target.cdr, input.cdr, path & sdiffPart(1), mismatches) + + of SNil: + discard + + of SKeyword: + aux(target.value, input.value, path, mismatches) + + + aux(target, input, @[], result) + +func formatPath(path: SexpPath): string = + ## Format S-expression path + if path.len == 0: + result = "" + + else: + for part in path: + case part.kind: + of spIndex: + result.add "[" & $part.index & "]" + + of spKey: + result.add ":" & part.key + +proc describeDiff*(diff: seq[SexpMismatch], conf: DiffFormatConf): ColText = + ## Generate colortext description of the S-expression mismatch diff + coloredResult() + + for idx, mismatch in diff: + if 0 < idx: + add "\n" + + add formatPath(mismatch.path) + fgYellow + case mismatch.kind: + of smKindMismatch: + addf( + "expected kind '$#', but got '$#'", + $mismatch.expected.kind + fgGreen, + $mismatch.found.kind + fgRed + ) + + of smMissingKey: + add " misses key ", mismatch.key + fgRed + + of smDifferentLiteral, smDifferentSymbol: + let exp = $mismatch.expected + let got = $mismatch.found + addf(" expected $#, but got $#", exp + fgGreen, got + fgRed) + if '\n' notin exp and '\n' notin got: + addf(" ($#)", formatInlineDiff(exp, got, conf)) + + of smArrayLen: + addf( + " len mismatch. Expected $# elements, but got $#", + $mismatch.expected.len + fgGreen, + $mismatch.found.len + fgRed + ) + +proc toLine*(s: SexpNode, sortfield: bool = false): ColText = + ## Generate colored formatting of the S-expression. + ## + ## - `sortfield` - order SKeyword entries in lists by the key name + coloredResult() + + let dim = styleDim + proc aux(s: SexpNode) = + if s.isNil: return + case s.kind: + of SInt: add $s.getNum() + fgCyan + of SString: add ("\"" & s.getStr() & "\"") + fgYellow + of SFloat: add $s.getFNum() + fgMagenta + of SNil: add "nil" + of SSymbol: add s.getSymbol() + fgCyan + of SCons: + add "(" + dim + aux(s.car) + add " . " + dim + aux(s.cdr) + add ")" + dim + of SKeyword: + add ":" + fgBlue + add s.getKey() + fgBlue + add " " + aux(s.value) + + of SList: + add "(" + dim + var first = true + if sortfield: + var fieldIdx: seq[(int, string)] + for idx, item in pairs(s): + if item.kind == SKeyword: + fieldIdx.add (idx, item.getKey()) + + let sortedFields = fieldIdx.sortedByIt(it[1]) + var nameIdx = 0 + for item in items(s): + if not first: add " " + if item.kind == SKeyword: + aux(s[sortedFields[nameIdx][0]]) + inc nameIdx + + else: + aux(item) + + first = false + + else: + for item in items(s): + if not first: add " " + first = false + aux(item) + + add ")" + dim + + aux(s) + + +# when isMainModule: +# let s = @[ +# "(:a b :c d)", +# "(:c d :a b)" +# ] + +# for item in s: +# echo item.parseSexp().toLine(sortfield = true) + +# when isMainModule and false: +# for str in @[ +# ("1", "2"), +# ("(:line 12 :col 10)", "(:line 30 :col 30)"), +# ("(Kind :expr 12)", "(Kind :expr 39)"), +# ("(Kind :expr 12)", "(Kind)"), +# ("(SymA :expr 12)", "(SymB :expr 12)") +# ]: +# let diff = sdiff(str[0], str[1]) +# if diff.isSome(): +# echo "```diff" +# echo "- ", str[0] +# echo "+ ", str[1] +# echo diff.get() +# echo "```\n" diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 873949ca674..54d4568126f 100644 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -2660,41 +2660,73 @@ func findNormalized(x: string, inArray: openArray[string]): int = # security hole... return -1 -func invalidFormatString() {.noinline.} = - raise newException(ValueError, "invalid format string") +func invalidFormatString(explain: string) {.noinline.} = + ## Raise value erorr for invalid interpolation format string + raise newException(ValueError, "invalid format string - " & explain) -func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, - extern: "nsuAddf".} = - ## The same as `add(s, formatstr % a)`, but more efficient. - const PatternChars = {'a'..'z', 'A'..'Z', '0'..'9', '\128'..'\255', '_'} +type + AddfFragmentKind* = enum + ## Kind of the `addf` interpolation fragment + addfText ## Regular text fragment + addfPositional ## Positional fragment `$#` + addfIndexed ## Indexed fragment `$1` + addfDollar ## Dollar literal `$$` + addfBackIndexed ## Negative indexed fragment `$-1` + addfVar ## Interpolated variable `$name` + addfExpr ## Expression in braces `${some expr}` + + AddfFragment* = object + ## `addf` format string fragment - can be used to write own text + ## interpolation logic. + case kind*: AddfFragmentKind + of addfText, addfVar, addfExpr: + text*: string + + of addfIndexed, addfPositional, addfBackIndexed: + idx*: int + + of addfDollar: + discard + +iterator addfFragments*(formatstr: string): AddfFragment = + ## Iterate over interpolation fragments of the `formatstr` var i = 0 var num = 0 + const PatternChars = {'a'..'z', 'A'..'Z', '0'..'9', '\128'..'\255', '_'} while i < len(formatstr): + var frag: AddfFragment if formatstr[i] == '$' and i+1 < len(formatstr): case formatstr[i+1] of '#': - if num > a.high: invalidFormatString() - add s, a[num] + frag = AddfFragment(kind: addfIndexed, idx: num) inc i, 2 inc num + of '$': - add s, '$' inc(i, 2) + frag = AddfFragment(kind: addfDollar) + of '1'..'9', '-': var j = 0 inc(i) # skip $ + let starti = i var negative = formatstr[i] == '-' if negative: inc i while i < formatstr.len and formatstr[i] in Digits: j = j * 10 + ord(formatstr[i]) - ord('0') inc(i) - let idx = if not negative: j-1 else: a.len-j - if idx < 0 or idx > a.high: invalidFormatString() - add s, a[idx] + + if negative: + frag = AddfFragment(kind: addfBackIndexed, idx: j) + + else: + frag = AddfFragment(kind: addfIndexed, idx: j - 1) + of '{': var j = i+2 var k = 0 var negative = formatstr[j] == '-' + let starti = j if negative: inc j var isNumber = 0 while j < formatstr.len and formatstr[j] notin {'\0', '}'}: @@ -2705,27 +2737,73 @@ func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, isNumber = -1 inc(j) if isNumber == 1: - let idx = if not negative: k-1 else: a.len-k - if idx < 0 or idx > a.high: invalidFormatString() - add s, a[idx] + if negative: + frag = AddfFragment(kind: addfBackIndexed, idx: k) + + else: + frag = AddfFragment(kind: addfIndexed, idx: k - 1) + else: - var x = findNormalized(substr(formatstr, i+2, j-1), a) - if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() - i = j+1 + frag = AddfFragment(kind: addfExpr, text: substr(formatstr, i+2, j-1)) + + i = j + 1 + of 'a'..'z', 'A'..'Z', '\128'..'\255', '_': var j = i+1 - while j < formatstr.len and formatstr[j] in PatternChars: inc(j) - var x = findNormalized(substr(formatstr, i+1, j-1), a) - if x >= 0 and x < high(a): add s, a[x+1] - else: invalidFormatString() + while j < formatstr.len and formatstr[j] in PatternChars: + inc(j) + + frag = AddfFragment(kind: addfVar, text: substr(formatstr, i+1, j-1)) + i = j + else: - invalidFormatString() + invalidFormatString("unexpected char after $ - " & $formatstr[i + 1]) + else: - add s, formatstr[i] + var trange = i .. i + while trange.b < formatstr.len and formatstr[trange.b] != '$': + inc trange.b + + dec trange.b + + frag = AddfFragment(kind: addfText, text: formatstr[trange]) + + i = trange.b inc(i) + yield frag + + + +func addf*(s: var string, formatstr: string, a: varargs[string, `$`]) {.rtl, + extern: "nsuAddf".} = + ## The same as `add(s, formatstr % a)`, but more efficient. + for fr in addfFragments(formatstr): + case fr.kind: + of addfDollar: + s.add '$' + + of addfPositional, addfIndexed, addfBackIndexed: + let idx = if fr.kind == addfBackIndexed: len(a) - fr.idx else: fr.idx + if not (0 <= idx and idx < a.len): + invalidFormatString( + "index for " & $idx & " is out of bounds for format arguments") + + s.add a[idx] + + of addfText: + s.add fr.text + + of addfVar, addfExpr: + var x = findNormalized(fr.text, a) + if x >= 0 and x < high(a): + add s, a[x+1] + + else: + invalidFormatString(" no named interpolation argument") + + func `%`*(formatstr: string, a: openArray[string]): string {.rtl, extern: "nsuFormatOpenArray".} = ## Interpolates a format string with the values from `a`. @@ -2893,3 +2971,10 @@ func isEmptyOrWhitespace*(s: string): bool {.rtl, extern: "nsuIsEmptyOrWhitespace".} = ## Checks if `s` is empty or consists entirely of whitespace characters. result = s.allCharsInSet(Whitespace) + +when isMainModule: + for fr in addfFragments("$# $1 ${name} $-1 $4"): + echo fr + + echo "$1" % "'_" + echo "$1 $1" % "'_" diff --git a/nimsuggest/nimsuggest.nim b/nimsuggest/nimsuggest.nim index 2c3460f7d6b..c57443deecc 100644 --- a/nimsuggest/nimsuggest.nim +++ b/nimsuggest/nimsuggest.nim @@ -13,7 +13,7 @@ when not defined(nimcore): {.error: "nimcore MUST be defined for Nim's core tooling".} import std/[strutils, os, parseopt, parseutils, sequtils, net, rdstdin] -import sexp +import experimental/sexp import std/options as std_options # Do NOT import suggest. It will lead to weird bugs with diff --git a/nimsuggest/tester.nim b/nimsuggest/tester.nim index 0b4d88bfade..96fd9b596b3 100644 --- a/nimsuggest/tester.nim +++ b/nimsuggest/tester.nim @@ -5,7 +5,8 @@ # When debugging, to run a single test, use for e.g.: # `nim r nimsuggest/tester.nim nimsuggest/tests/tsug_accquote.nim` -import os, osproc, strutils, streams, re, sexp, net +import std/[os, osproc, strutils, streams, re, net] +import experimental/sexp from sequtils import toSeq type diff --git a/testament/backend.nim b/testament/backend.nim index e375d0c230b..8796f4215fc 100644 --- a/testament/backend.nim +++ b/testament/backend.nim @@ -50,6 +50,9 @@ proc writeTestResult*(name, category, target, action, result, expected, given: s currentCategory = category results = newJArray() + if results.isNil(): + results = newJArray() + results.add %*{"name": name, "category": category, "target": target, "action": action, "result": result, "expected": expected, "given": given, "machine": thisMachine.string, "commit": thisCommit.string, "branch": thisBranch} @@ -99,7 +102,7 @@ proc cacheResults*() = let noderesult = node{"result"}.getStr() if not passResults.contains(noderesult): fresults.add(node) - + var results = open("testresults" / "cacheresults" / "result".addFileExt"json", fmWrite) results.write(fresults.pretty()) close(results) @@ -151,4 +154,4 @@ proc getRetries*(): (seq[string], seq[(string, string)]) = # In the case that rsplit did not separate the name into a target and # test name then we still add it else: (name_target[0], "") - result = (cats, file_targs) \ No newline at end of file + result = (cats, file_targs) diff --git a/testament/lib/stdtest/specialpaths.nim b/testament/lib/stdtest/specialpaths.nim index 53b94fdbb0f..ca7116f1f2e 100644 --- a/testament/lib/stdtest/specialpaths.nim +++ b/testament/lib/stdtest/specialpaths.nim @@ -46,7 +46,8 @@ proc splitTestFile*(file: string): tuple[cat: string, path: string] = else: result.path = file return result - doAssert false, "file must match this pattern: '/pathto/tests/dir/**/tfile.nim', got: '" & file & "'" + + result.path = file static: # sanity check diff --git a/testament/specs.nim b/testament/specs.nim index 1293d5190bf..514195f3266 100644 --- a/testament/specs.nim +++ b/testament/specs.nim @@ -76,35 +76,41 @@ type TSpec* = object # xxx make sure `isJoinableSpec` takes into account each field here. - # description*: string ## document the purpose of the test + description*: string ## document the purpose of the test action*: TTestAction - file*, cmd*: string - input*: string - outputCheck*: TOutputCheck - sortoutput*: bool - output*: string + file*: string ## File that test spec has been parsed from + cmd*: string ## Command to execute for the test + input*: string ## `stdin` input that will be piped into program after + ## it has been compiled. + outputCheck*: TOutputCheck ## Kind of the output checking for the + ## executed compiled program. + sortoutput*: bool ## Sort runtime output + output*: string ## Expected runtime output line*, column*: int exitCode*: int msg*: string - ccodeCheck*: seq[string] - maxCodeSize*: int + ccodeCheck*: seq[string] ## List of peg patterns that need to be + ## searched for in the generated code. Used for backend code testing. + maxCodeSize*: int ## Maximum allowed code size (in bytes) for the test. err*: TResultEnum inCurrentBatch*: bool targets*: set[TTarget] matrix*: seq[string] + nimoutSexp*: bool nimout*: string - nimoutFull*: bool # whether nimout is all compiler output or a subset - parseErrors*: string # when the spec definition is invalid, this is not empty. + nimoutFull*: bool ## whether nimout is all compiler output or a subset + parseErrors*: string ## when the spec definition is invalid, this is + ## not empty. unjoinable*: bool unbatchable*: bool - # whether this test can be batchable via `NIM_TESTAMENT_BATCH`; only very - # few tests are not batchable; the ones that are not could be turned batchable - # by making the dependencies explicit + ## whether this test can be batchable via `NIM_TESTAMENT_BATCH`; only + ## very few tests are not batchable; the ones that are not could be + ## turned batchable by making the dependencies explicit useValgrind*: ValgrindSpec - timeout*: float # in seconds, fractions possible, - # but don't rely on much precision - inlineErrors*: seq[InlineError] # line information to error message - debugInfo*: string # debug info to give more context + timeout*: float ## in seconds, fractions possible, but don't rely on + ## much precision + inlineErrors*: seq[InlineError] ## line information to error message + debugInfo*: string ## debug info to give more context knownIssues*: seq[string] ## known issues to be fixed RetryContainer* = object @@ -118,6 +124,7 @@ type var retryContainer* = RetryContainer(retry: false) proc getCmd*(s: TSpec): string = + ## Get runner command for a given test specification if s.cmd.len == 0: result = compilerPrefix & " $target --hints:on -d:testing --clearNimblePath --nimblePath:build/deps/pkgs $options $file" else: @@ -158,6 +165,7 @@ const inlineErrorMarker = "#[tt." proc extractErrorMsg(s: string; i: int; line: var int; col: var int; spec: var TSpec): int = + ## Get position of the error message in input text `s`. result = i + len(inlineErrorMarker) inc col, len(inlineErrorMarker) var kind = "" @@ -218,13 +226,22 @@ proc extractSpec(filename: string; spec: var TSpec): string = var col = 1 while i < s.len: if (i == 0 or s[i-1] != ' ') and s.continuesWith(specStart, i): - # `s[i-1] == '\n'` would not work because of `tests/stdlib/tbase64.nim` which contains BOM (https://en.wikipedia.org/wiki/Byte_order_mark) + # `s[i-1] == '\n'` would not work because of + # `tests/stdlib/tbase64.nim` which contains BOM + # (https://en.wikipedia.org/wiki/Byte_order_mark) const lineMax = 10 if a != -1: - raise newException(ValueError, "testament spec violation: duplicate `specStart` found: " & $(filename, a, b, line)) + raise newException( + ValueError, + "testament spec violation: duplicate `specStart` found: " & + $(filename, a, b, line)) elif line > lineMax: - # not overly restrictive, but prevents mistaking some `specStart` as spec if deeep inside a test file - raise newException(ValueError, "testament spec violation: `specStart` should be before line $1, or be indented; info: $2" % [$lineMax, $(filename, a, b, line)]) + # not overly restrictive, but prevents mistaking some `specStart` + # as spec if deeep inside a test file + raise newException( + ValueError, + "testament spec violation: `specStart` should be before line $1, or be indented; info: $2" % [ + $lineMax, $(filename, a, b, line)]) i += specStart.len a = i elif a > -1 and b == -1 and s.continuesWith(tripleQuote, i): @@ -235,6 +252,7 @@ proc extractSpec(filename: string; spec: var TSpec): string = inc i col = 1 elif s.continuesWith(inlineErrorMarker, i): + # Found `#[tt.` - extract it i = extractErrorMsg(s, i, line, col, spec) else: inc col @@ -243,11 +261,15 @@ proc extractSpec(filename: string; spec: var TSpec): string = if a >= 0 and b > a: result = s.substr(a, b-1).multiReplace({"'''": tripleQuote, "\\31": "\31"}) elif a >= 0: - raise newException(ValueError, "testament spec violation: `specStart` found but not trailing `tripleQuote`: $1" % $(filename, a, b, line)) + raise newException( + ValueError, + "testament spec violation: `specStart` found but not trailing `tripleQuote`: $1" % + $(filename, a, b, line)) else: result = "" proc parseTargets*(value: string): set[TTarget] = + ## Get list of allowed run targets for the testament for v in value.normalize.splitWhitespace: case v of "c": result.incl(targetC) @@ -275,6 +297,7 @@ proc isCurrentBatch*(testamentData: TestamentData; filename: string): bool = true proc parseSpec*(filename: string): TSpec = + ## Extract and parse specification for a given file path result.file = filename when defined(windows): @@ -308,6 +331,17 @@ proc parseSpec*(filename: string): TSpec = # incorporate it into the the actual test runner and output. # result.description = e.value discard + of "nimoutformat": + case e.value.normalize: + of "sexp": + result.nimoutSexp = true + + of "text": + result.nimoutSexp = false + + else: + result.parseErrors.addLine "unexpected nimout format: got ", e.value + of "action": case e.value.normalize of "compile": @@ -432,7 +466,7 @@ proc parseSpec*(filename: string): TSpec = of "knownissue": case e.value.normalize of "n", "no", "false", "0": discard - else: + else: result.knownIssues.add e.value result.err = reDisabled else: diff --git a/testament/testament.nim b/testament/testament.nim index 747952505d0..08b1fad81b7 100644 --- a/testament/testament.nim +++ b/testament/testament.nim @@ -11,14 +11,15 @@ import std/[ strutils, pegs, os, osproc, streams, json, exitprocs, parseopt, browsers, - terminal, algorithm, times, md5, intsets, macros + terminal, algorithm, times, md5, intsets, macros, tables, + options ] import backend, azure, htmlgen, specs from std/sugar import dup import utils/nodejs import lib/stdtest/testutils from lib/stdtest/specialpaths import splitTestFile -from std/private/gitutils import diffStrings +import experimental/[sexp, sexp_diff, colortext, colordiff] proc trimUnitSep(x: var string) = let L = x.len @@ -32,10 +33,110 @@ var optVerbose = false var useMegatest = true var optFailing = false +import std/sugar + +type + TOutReport = object + inline: Option[InlineError] + node: SexpNode + file: string + + TOutCompare = ref object + ## Result of comparing two data outputs for a given spec + match: bool + expectedReports: seq[TOutReport] + givenReports: seq[TOutReport] + sortedMapping: seq[tuple[pair: (int, int), cost: int]] + diffMap: Table[(int, int), seq[SexpMismatch]] + ignoredExpected: seq[int] + ignoredGiven: seq[int] + cantIgnoreGiven: bool + + + +proc diffStrings*(a, b: string): tuple[output: string, same: bool] = + let a = a.split("\n") + let b = b.split("\n") + var maxA = 0 + var maxB = 0 + for line in a: + maxA = max(maxA, line.len) + + for line in b: + maxB = max(maxB, line.len) + + var conf = diffFormatter() + conf.sideBySide = maxA + maxB + 8 < terminalWidth() + conf.groupLine = true + + let diff = myersDiff(a, b) + if len(diff) == 0: + result.same = true + + else: + result.same = false + result.output = diff.shiftDiffed(a, b). + formatDiffed(a, b, conf).toString(useColors) + +proc format(tcmp: TOutCompare): ColText = + ## Pretty-print structured output comparison for further printing. + var + conf = diffFormatter() + res: ColText + + coloredResult() + + var first = true + proc addl() = + if not first: + add "\n" + + first = false + + for (pair, weight) in tcmp.sortedMapping: + if 0 < weight: + addl() + addl() + let exp = tcmp.expectedReports[pair[0]] + add "Expected" + if exp.inline.isSome(): + let inline = exp.inline.get() + addf(" inline $# annotation at $#($#, $#)", + inline.kind + fgGreen, + exp.file + fgYellow, + $inline.line + fgCyan, + $inline.col + fgCyan + ) + + addf(":\n\n- $#\n\nGiven:\n\n+ $#\n\n", + exp.node.toLine(sortfield = true), + tcmp.givenReports[pair[1]].node.toLine(sortfield = true) + ) + + add tcmp.diffMap[pair].describeDiff(conf).indent(2) + + + for exp in tcmp.ignoredExpected: + addl() + addl() + addf( + "Missing expected annotation:\n\n? $#\n\n", + tcmp.expectedReports[exp].node.toLine(sortfield = true) + ) + + if tcmp.cantIgnoreGiven: + for give in tcmp.ignoredGiven: + addl() + addl() + addf( + "Unexpected given annotation:\n\n? $#\n\n", + tcmp.expectedReports[give].node.toLine(sortfield = true) + ) + ## Blanket method to encaptsulate all echos while testament is detangled. -## Using this means echo cannot be called with separation of args and must instead -## pass a single concatenated string so that optional paramters can be -## included +## Using this means echo cannot be called with separation of args and must +## instead pass a single concatenated string so that optional paramters can +## be included type MessageType = enum Undefined, @@ -125,12 +226,13 @@ var gTargets = {low(TTarget)..high(TTarget)} var targetsSet = false proc isSuccess(input: string): bool = - # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in pegs - # note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; while we - # could handle colors, there would be other issues such as handling other flags - # that may appear in user config (eg: `--filenames`). - # Passing `XDG_CONFIG_HOME= testament args...` can be used to ignore user config - # stored in XDG_CONFIG_HOME, refs https://wiki.archlinux.org/index.php/XDG_Base_Directory + # not clear how to do the equivalent of pkg/regex's: re"FOO(.*?)BAR" in + # pegs note: this doesn't handle colors, eg: `\e[1m\e[0m\e[32mHint:`; + # while we could handle colors, there would be other issues such as + # handling other flags that may appear in user config (eg: + # `--filenames`). Passing `XDG_CONFIG_HOME= testament args...` can be + # used to ignore user config stored in XDG_CONFIG_HOME, refs + # https://wiki.archlinux.org/index.php/XDG_Base_Directory input.startsWith("Hint: ") and input.endsWith("[SuccessX]") proc getFileDir(filename: string): string = @@ -188,6 +290,11 @@ proc prepareTestCmd(cmdTemplate, filename, options, nimcache: string, proc callNimCompiler(cmdTemplate, filename, options, nimcache: string, target: TTarget, extraOptions = ""): TSpec = + ## Execute nim compiler with given `filename`, `options` and `nimcache`. + ## Compile to target specified in the `target` and return compilation + ## results as a new `TSpec` value. Resulting spec contains `.nimout` set + ## from the compiler run results as well as known inline messages (output + ## is immedately scanned for results). result.cmd = prepareTestCmd(cmdTemplate, filename, options, nimcache, target, extraOptions) verboseCmd(result.cmd) @@ -280,88 +387,190 @@ Tests failed and allowed to fail: $3 / $1
Tests skipped: $4 / $1
""" % [$x.total, $x.passed, $x.failedButAllowed, $x.skipped] -proc addResult(r: var TResults, test: TTest, target: TTarget, - expected, given: string, successOrig: TResultEnum, allowFailure = false, givenSpec: ptr TSpec = nil) = - # instead of `ptr TSpec` we could also use `Option[TSpec]`; passing `givenSpec` makes it easier to get what we need - # instead of having to pass individual fields, or abusing existing ones like expected vs given. - # test.name is easier to find than test.name.extractFilename - # A bit hacky but simple and works with tests/testament/tshould_not_work.nim +proc getName(test: TTest, target: TTarget, allowFailure: bool): string = var name = test.name.replace(DirSep, '/') name.add ' ' & $target if allowFailure: name.add " (allowed to fail) " - if test.options.len > 0: name.add ' ' & test.options + if test.options.len > 0: + name.add ' ' & test.options - let duration = epochTime() - test.startTime - let success = if test.spec.timeout > 0.0 and duration > test.spec.timeout: reTimeout - else: successOrig + return name + + +type + ReportParams = object + ## Contains additional data about report execution state. + duration: float + name: string + outCompare: TOutCompare + success: TResultEnum + + expected, given: string + +proc logToConsole( + test: TTest, + param: ReportParams, + givenSpec: ptr TSpec = nil + ) = + + ## Format test infomation to the console. `test` contains information + ## about the test itself, `param` contains additional data about test + ## execution. + + let durationStr = param.duration.formatFloat( + ffDecimal, precision = 2).align(5) - let durationStr = duration.formatFloat(ffDecimal, precision = 2).align(5) - if backendLogging: - backend.writeTestResult(name = name, - category = test.cat.string, - target = $target, - action = $test.spec.action, - result = $success, - expected = expected, - given = given) - r.data.addf("$#\t$#\t$#\t$#", name, expected, given, $success) template dispNonSkipped(color, outcome) = if not optFailing or color == fgRed: - maybeStyledEcho color, outcome, fgCyan, test.debugInfo, alignLeft(name, 60), fgBlue, " (", durationStr, " sec)" + maybeStyledEcho( + color, outcome, fgCyan, test.debugInfo, alignLeft(param.name, 60), + fgBlue, " (", durationStr, " sec)") + template disp(msg) = if not optFailing: - maybeStyledEcho styleDim, fgYellow, msg & ' ', styleBright, fgCyan, name - if success == reSuccess: + maybeStyledEcho( + styleDim, fgYellow, msg & ' ', styleBright, fgCyan, param.name) + + if param.success == reSuccess: dispNonSkipped(fgGreen, "PASS: ") - elif success == reDisabled: - if test.spec.inCurrentBatch: disp("SKIP:") - else: disp("NOTINBATCH:") - elif success == reJoined: disp("JOINED:") + + elif param.success == reDisabled: + if test.spec.inCurrentBatch: + disp("SKIP:") + + else: + disp("NOTINBATCH:") + + elif param.success == reJoined: + disp("JOINED:") + else: dispNonSkipped(fgRed, failString) - maybeStyledEcho styleBright, fgCyan, "Test \"", test.name, "\"", " in category \"", test.cat.string, "\"" - maybeStyledEcho styleBright, fgRed, "Failure: ", $success + maybeStyledEcho( + styleBright, fgCyan, "Test \"", test.name, "\"", + " in category \"", test.cat.string, "\"") + + maybeStyledEcho styleBright, fgRed, "Failure: ", $param.success if givenSpec != nil and givenSpec.debugInfo.len > 0: msg Undefined: "debugInfo: " & givenSpec.debugInfo - if success in {reBuildFailed, reNimcCrash, reInstallFailed}: + + if param.success in {reBuildFailed, reNimcCrash, reInstallFailed}: # expected is empty, no reason to print it. - msg Undefined: given + msg Undefined: param.given + else: - maybeStyledEcho fgYellow, "Expected:" - maybeStyledEcho styleBright, expected, "\n" - maybeStyledEcho fgYellow, "Gotten:" - maybeStyledEcho styleBright, given, "\n" - msg Undefined: diffStrings(expected, given).output + if not isNil(param.outCompare): + msg Undefined: + param.outCompare.format().toString(useColors) - if backendLogging and (isAppVeyor or isAzure): - let (outcome, msg) = - case success - of reSuccess: - ("Passed", "") - of reDisabled, reJoined: - ("Skipped", "") - of reBuildFailed, reNimcCrash, reInstallFailed: - ("Failed", "Failure: " & $success & '\n' & given) else: - ("Failed", "Failure: " & $success & "\nExpected:\n" & expected & "\n\n" & "Gotten:\n" & given) - if isAzure: - azure.addTestResult(name, test.cat.string, int(duration * 1000), msg, success) + # REFACTOR error message formatting should be based on the + # `TestReport` data structure that contains all the necessary + # inforamtion that is necessary in order to generate error message. + maybeStyledEcho fgYellow, "Expected:" + maybeStyledEcho styleBright, param.expected, "\n" + maybeStyledEcho fgYellow, "Gotten:" + maybeStyledEcho styleBright, param.given, "\n" + msg Undefined: + diffStrings(param.expected, param.given).output + + +proc logToBackend( + test: TTest, + param: ReportParams + ) = + + let (outcome, msg) = + case param.success + of reSuccess: + ("Passed", "") + of reDisabled, reJoined: + ("Skipped", "") + of reBuildFailed, reNimcCrash, reInstallFailed: + ("Failed", "Failure: " & $param.success & '\n' & param.given) else: - var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options, - "-Framework", "nim-testament", "-FileName", - test.cat.string, - "-Outcome", outcome, "-ErrorMessage", msg, - "-Duration", $(duration * 1000).int], - options = {poStdErrToStdOut, poUsePath, poParentStreams}) - discard waitForExit(p) - close(p) + ("Failed", "Failure: " & $param.success & "\nExpected:\n" & + param.expected & "\n\n" & "Gotten:\n" & param.given) + if isAzure: + azure.addTestResult( + param.name, test.cat.string, int(param.duration * 1000), msg, param.success) + + else: + var p = startProcess("appveyor", args = ["AddTest", test.name.replace("\\", "/") & test.options, + "-Framework", "nim-testament", "-FileName", + test.cat.string, + "-Outcome", outcome, "-ErrorMessage", msg, + "-Duration", $(param.duration * 1000).int], + options = {poStdErrToStdOut, poUsePath, poParentStreams}) + discard waitForExit(p) + close(p) + + +proc addResult( + r: var TResults, + test: TTest, + target: TTarget, + expected, given: string, + successOrig: TResultEnum, + allowFailure = false, + givenSpec: ptr TSpec = nil, + outCompare: TOutCompare = nil + ) = + ## Report final test results to backend, end user (write to command-line) + ## and so on. + # instead of `ptr tspec` we could also use `option[tspec]`; passing + # `givenspec` makes it easier to get what we need instead of having to + # pass individual fields, or abusing existing ones like expected vs + # given. test.name is easier to find than test.name.extractfilename a bit + # hacky but simple and works with tests/testament/tshould_not_work.nim + + # Compute test test duration, final success status, prepare formatting variables + var param: ReportParams + + param.expected = expected + param.given = given + param.outCompare = outCompare + param.duration = epochTime() - test.startTime + param.success = + if test.spec.timeout > 0.0 and param.duration > test.spec.timeout: + reTimeout + else: + successOrig + + + param.name = test.getName(target, allowFailure) + + if backendLogging: + backend.writeTestResult(name = param.name, + category = test.cat.string, + target = $target, + action = $test.spec.action, + result = $param.success, + expected = expected, + given = given) + + # TODO DOC what is this + r.data.addf("$#\t$#\t$#\t$#", param.name, expected, given, $param.success) + + + # Write to console + logToConsole(test, param, givenSpec) + + if backendLogging and (isAppVeyor or isAzure): + # Write to logger + logToBackend(test, param) proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = + ## Check for inline error annotations in the nimout results, comparing + ## them with the output of the compiler. + let pegLine = peg"{[^(]*} '(' {\d+} ', ' {\d+} ') ' {[^:]*} ':' \s* {.*}" var covered = initIntSet() for line in splitLines(given.nimout): + # Iterate over each line in the output + # Searching for the `file(line, col) Severity: text` pattern if line =~ pegLine: let file = extractFilename(matches[0]) let line = try: parseInt(matches[1]) except: -1 @@ -370,26 +579,33 @@ proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, let msg = matches[4] if file == extractFilename test.name: + # If annotation comes from the target file var i = 0 for x in expected.inlineErrors: if x.line == line and (x.col == col or x.col < 0) and x.kind == kind and x.msg in msg: + # And annotaiton has matching line, column and message + # information, register it as 'covered' covered.incl i inc i block coverCheck: for j in 0..high(expected.inlineErrors): + # For each output message that was not covered by annotations, add it + # to the output as 'missing' if j notin covered: - var e = test.name + var e: string + let exp = expected.inlineErrors[j] + e = test.name e.add '(' - e.addInt expected.inlineErrors[j].line - if expected.inlineErrors[j].col > 0: + e.addInt exp.line + if exp.col > 0: e.add ", " - e.addInt expected.inlineErrors[j].col + e.addInt exp.col e.add ") " - e.add expected.inlineErrors[j].kind + e.add exp.kind e.add ": " - e.add expected.inlineErrors[j].msg + e.add exp.msg r.addResult(test, target, e, given.nimout, reMsgsDiffer) break coverCheck @@ -398,6 +614,8 @@ proc checkForInlineErrors(r: var TResults, expected, given: TSpec, test: TTest, inc(r.passed) proc nimoutCheck(expected, given: TSpec): bool = + ## Check if expected nimout values match with specified ones. This check + ## implements comparison of the unstructured data. result = true if expected.nimoutFull: if expected.nimout != given.nimout: @@ -405,25 +623,141 @@ proc nimoutCheck(expected, given: TSpec): bool = elif expected.nimout.len > 0 and not greedyOrderedSubsetLines(expected.nimout, given.nimout): result = false -proc cmpMsgs(r: var TResults, expected, given: TSpec, test: TTest, target: TTarget) = - if expected.inlineErrors.len > 0: +proc sexpCheck(test: TTest, expected, given: TSpec): TOutCompare = + ## Check if expected nimout values match with specified ones. Thish check + ## implements a structured comparison of the data and returns full report + ## about all the mismatches that can be formatted as needed. + ## This procedure determines whether `given` spec matches `expected` test + ## results. + var r = TOutCompare() + r.cantIgnoreGiven = expected.nimoutFull + + for exp in expected.inlineErrors: + var parsed = parseSexp(exp.msg) + var loc = convertSexp([sexp(test.name), sexp(exp.line)]) + if exp.col > 0: + loc.add sexp(exp.col) + + parsed.addField("location", loc) + parsed.addField("severity", newSSymbol(exp.kind)) + r.expectedReports.add TOutReport(inline: some exp, node: parsed, file: expected.file) + + for line in splitLines(expected.nimout): + if 0 < line.len: + r.expectedReports.add TOutReport(node: parseSexp(line)) + + for line in splitLines(given.nimout): + if 0 < line.len: + r.givenReports.add TOutReport(node: parseSexp(line)) + + var map = r.diffMap + proc reportCmp(a, b: int): int = + # Best place for further optimization and configuration - if more + # comparison speed isneeded, try starting with error kind, file, line + # comparison, they doing a regular msg != msg compare and only then + # deep structural diff. + if r.expectedReports[a].node[0] != r.givenReports[b].node[0]: + result += 10 + + let diff = diff(r.expectedReports[a].node, r.givenReports[b].node) + r.diffMap[(a, b)] = diff + result += diff.len + + (r.ignoredExpected, r.ignoredGiven, r.sortedMapping) = stablematch( + r.expectedReports.len, + r.givenReports.len, + reportCmp, + Descending + ) + + if 0 < r.sortedMapping[0].cost: + r.match = false + + elif 0 < r.ignoredGiven.len and expected.nimoutFull: + r.match = false + + return r + +proc cmpMsgs( + r: var TResults, expected, given: TSpec, test: TTest, target: TTarget + ) = + ## Compare all test output messages. This proc does structured or + ## unstructured comparison comparison and immediately reports it's + ## results. + ## + ## It is used to for performting 'reject' action checks - it compares + ## both inline and regular messages - in addition to `nimoutCheck` + + # If structural comparison is requested - drop directly to it and handle + # the success/failure modes in the branch + if expected.nimoutSexp: + let outCompare = test.sexpCheck(expected, given) + # Full match of the output results. + if outCompare.match: + r.addResult(test, target, expected.msg, given.msg, reSuccess) + inc(r.passed) + + else: + # Write out error message. + r.addResult( + test, target, expected.msg, given.msg, reMsgsDiffer, + givenSpec = unsafeAddr given, + outCompare = outCompare + ) + + + # Checking for inline errors. + elif expected.inlineErrors.len > 0: + # QUESTION - `checkForInlineErrors` does not perform any comparisons + # for the regular message spec, it just compares annotated messages. + # How can it report anything properly then? + # + # MAYBE it is related the fact testament misuses the `inlineErrors`, + # and wrongly assumes they are /only/ errors, despite actually parsing + # anything that starts with `#[tt.` as inline annotation? Even in this + # case this does not make any sense, because comparisons is done only + # for inline error messages. + # + # MAYBE this is just a way to mitigate the more complex problem of + # mixing in inline error messages and regular `.nimout`? I 'solved' it + # using `stablematch` and weighted ordering, so most likely the person + # who wrote this solved the same problem using "I don't care" approach. + # + # https://github.com/nim-lang/Nim/commit/9a110047cbe2826b1d4afe63e3a1f5a08422b73f#diff-a161d4667e86146f2f8003f08f765b8d9580ae92ec5fb6679c80c07a5310a551R362-R364 checkForInlineErrors(r, expected, given, test, target) + + # Check for `.errormsg` in expected and given spec first elif strip(expected.msg) notin strip(given.msg): r.addResult(test, target, expected.msg, given.msg, reMsgsDiffer) + + # Compare expected and resulted spec messages elif not nimoutCheck(expected, given): + # Report general message mismatch error r.addResult(test, target, expected.nimout, given.nimout, reMsgsDiffer) + + # Check for filename mismatches elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: + # Report error for the the error file mismatch r.addResult(test, target, expected.file, given.file, reFilesDiffer) - elif expected.line != given.line and expected.line != 0 or - expected.column != given.column and expected.column != 0: + + # Check for produced and given error message locations + elif expected.line != given.line and + expected.line != 0 or + expected.column != given.column and + expected.column != 0: + # Report error for the location mismatch r.addResult(test, target, $expected.line & ':' & $expected.column, $given.line & ':' & $given.column, reLinesDiffer) + + # None of the unstructructured checks found mismatches, reporting thest + # as passed. else: r.addResult(test, target, expected.msg, given.msg, reSuccess) inc(r.passed) proc generatedFile(test: TTest, target: TTarget): string = + ## Get path to the genrated file name from the test. if target == targetJS: result = test.name.changeFileExt("js") else: @@ -432,10 +766,19 @@ proc generatedFile(test: TTest, target: TTarget): string = result = nimcacheDir(test.name, test.options, target) / "@m" & name.changeFileExt(ext) proc needsCodegenCheck(spec: TSpec): bool = + ## If there is any checks that need to be performed for a generated code + ## file result = spec.maxCodeSize > 0 or spec.ccodeCheck.len > 0 -proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var string, - given: var TSpec) = +proc codegenCheck( + test: TTest, + target: TTarget, + spec: TSpec, + expectedMsg: var string, + given: var TSpec + ) = + ## Check for any codegen mismatches in file generated from `test` run. + ## Only file that was immediately generated is testsed. try: let genFile = generatedFile(test, target) let contents = readFile(genFile) @@ -460,20 +803,48 @@ proc codegenCheck(test: TTest, target: TTarget, spec: TSpec, expectedMsg: var st proc compilerOutputTests(test: TTest, target: TTarget, given: var TSpec, expected: TSpec; r: var TResults) = + ## Test output of the compiler for correctness var expectedmsg: string = "" var givenmsg: string = "" + var outCompare: TOutCompare if given.err == reSuccess: + # Check size??? of the generated C code. If fails then add error + # message. if expected.needsCodegenCheck: codegenCheck(test, target, expected, expectedmsg, given) givenmsg = given.msg - if not nimoutCheck(expected, given): - given.err = reMsgsDiffer - expectedmsg = expected.nimout - givenmsg = given.nimout.strip + + if expected.nimoutSexp: + # If test requires structural comparison - run it and then check + # output results for any failures. + outCompare = test.sexpCheck(expected, given) + if not outCompare.match: + given.err = reMsgsDiffer + + else: + # Use unstructured data comparison for the expected and given outputs + if not nimoutCheck(expected, given): + given.err = reMsgsDiffer + + # Just like unstructured comparison - assign expected/given pair. + # In that case deep structural comparison is not necessary so we + # are just pasing strings around, they will be diffed only on + # reporting. + expectedmsg = expected.nimout + givenmsg = given.nimout.strip + else: givenmsg = "$ " & given.cmd & '\n' & given.nimout - if given.err == reSuccess: inc(r.passed) - r.addResult(test, target, expectedmsg, givenmsg, given.err) + if given.err == reSuccess: + inc(r.passed) + + # Write out results of the compiler output testing + r.addResult( + test, target, expectedmsg, givenmsg, given.err, + givenSpec = addr given, + # Supply results of the optional structured comparison. + outCompare = outCompare + ) proc getTestSpecTarget(): TTarget = if getEnv("NIM_COMPILE_TO_CPP", "false") == "true": @@ -482,6 +853,8 @@ proc getTestSpecTarget(): TTarget = result = targetC proc checkDisabled(r: var TResults, test: TTest): bool = + ## Check if test has been enabled (not `disabled: true`, and not joined). + ## Return true if test can be executed. if test.spec.err in {reDisabled, reJoined}: # targetC is a lie, but parameter is required r.addResult(test, targetC, "", "", test.spec.err) @@ -494,16 +867,18 @@ proc checkDisabled(r: var TResults, test: TTest): bool = var count = 0 proc equalModuloLastNewline(a, b: string): bool = - # allow lazy output spec that omits last newline, but really those should be fixed instead + # allow lazy output spec that omits last newline, but really those should + # be fixed instead result = a == b or b.endsWith("\n") and a == b[0 ..< ^1] proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, target: TTarget, nimcache: string, extraOptions = "") = test.startTime = epochTime() template callNimCompilerImpl(): untyped = - # xxx this used to also pass: `--stdout --hint:Path:off`, but was done inconsistently - # with other branches - callNimCompiler(expected.getCmd, test.name, test.options, nimcache, target, extraOptions) + # xxx this used to also pass: `--stdout --hint:Path:off`, but was done + # inconsistently with other branches + callNimCompiler( + expected.getCmd, test.name, test.options, nimcache, target, extraOptions) case expected.action of actionCompile: var given = callNimCompilerImpl() @@ -511,7 +886,9 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, of actionRun: var given = callNimCompilerImpl() if given.err != reSuccess: - r.addResult(test, target, "", "$ " & given.cmd & '\n' & given.nimout, given.err, givenSpec = given.addr) + r.addResult( + test, target, "", "$ " & given.cmd & '\n' & given.nimout, + given.err, givenSpec = given.addr) else: let isJsTarget = target == targetJS var exeFile = changeFileExt(test.name, if isJsTarget: "js" else: ExeExt) @@ -521,8 +898,10 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, else: let nodejs = if isJsTarget: findNodeJs() else: "" if isJsTarget and nodejs == "": - r.addResult(test, target, expected.output, "nodejs binary not in PATH", - reExeNotFound) + r.addResult( + test, target, expected.output, + "nodejs binary not in PATH", reExeNotFound) + else: var exeCmd: string var args = test.testArgs @@ -538,9 +917,13 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, valgrindOptions.add "--leak-check=yes" args = valgrindOptions & exeCmd & args exeCmd = "valgrind" - var (_, buf, exitCode) = execCmdEx2(exeCmd, args, input = expected.input) - # Treat all failure codes from nodejs as 1. Older versions of nodejs used - # to return other codes, but for us it is sufficient to know that it's not 0. + + var (_, buf, exitCode) = execCmdEx2( + exeCmd, args, input = expected.input) + + # Treat all failure codes from nodejs as 1. Older versions of + # nodejs used to return other codes, but for us it is sufficient + # to know that it's not 0. if exitCode != 0: exitCode = 1 let bufB = if expected.sortoutput: @@ -555,14 +938,20 @@ proc testSpecHelper(r: var TResults, test: var TTest, expected: TSpec, r.addResult(test, target, "exitcode: " & $expected.exitCode, "exitcode: " & $exitCode & "\n\nOutput:\n" & bufB, reExitcodesDiffer) - elif (expected.outputCheck == ocEqual and not expected.output.equalModuloLastNewline(bufB)) or - (expected.outputCheck == ocSubstr and expected.output notin bufB): + elif ( + expected.outputCheck == ocEqual and + not expected.output.equalModuloLastNewline(bufB) + ) or ( + expected.outputCheck == ocSubstr and + expected.output notin bufB + ): given.err = reOutputsDiffer r.addResult(test, target, expected.output, bufB, reOutputsDiffer) else: compilerOutputTests(test, target, given, expected, r) of actionReject: let given = callNimCompilerImpl() + # Scan compiler output fully for all mismatches and report if any found cmpMsgs(r, expected, given, test, target) proc targetHelper(r: var TResults, test: TTest, expected: TSpec, extraOptions = "") = diff --git a/tests/stdlib/tcolordiff.nim b/tests/stdlib/tcolordiff.nim new file mode 100644 index 00000000000..b37895a3433 --- /dev/null +++ b/tests/stdlib/tcolordiff.nim @@ -0,0 +1,208 @@ +import experimental/[diff, colordiff, colortext] +import std/[strformat, strutils] + +# Configure diff formattter to use no unicode or colors to make testing +# easier. Each inserted/deleted chunk is annotated with `D/I/R/K` for the +# Delete/Insert/Replace/Keep respectively, and wrapped in the `[]`. Line +# split is done on each whitespace and elements are joined using `#` +# character. +# +# These test compare formatted edit operations - for now I've decided it +# is not necessary to factor out it out so much that we would be +# comparing raw layout data here (and considering most formatter procs +# already accept semi-ready input it would be a quite pointless +# indirection only for the purposes of testing) +# +# The strings here do not represent typical (default) formatting of the +# diff - formatting hooks were overriden to make the blocks more explicit +# instead. +var conf = diffFormatter(false) +conf.inlineDiffSeparator = clt("#") +conf.formatChunk = proc( + word: string, mode, secondary: SeqEditKind, inline: bool +): ColText = + if inline: + # Configure inline message diffing separately + if secondary == sekDelete: + &"[R/<{word}>-" + fgDefault + else: + &"<{word}>]" + fgDefault + else: + &"[{($mode)[3]}/{word}]" + fgDefault + +proc diff(a, b: string, sideBySide: bool = false): string = + conf.sideBySide = sideBySide + return formatDiffed(a, b, conf).toString(false) + +proc ediff(a, b: string): string = + formatInlineDiff(a, b, conf).toString(false) + +proc ldiff(a, b: string): (string, string) = + let (old, new) = formatLineDiff(a, b, conf) + return (old.toString(false), new.toString(false)) + +doAssert not hasInvisible(" a") +doAssert hasInvisible("a ") +doAssert hasInvisible("a \n") +doAssert not hasInvisible("a a") +doAssert hasInvisible("a\n") + +proc assertEq(found, expected: string) = + if found != expected: + assert false, &"expected:\n{expected}\nfound:\n{found}\ndiff:\n{diffText(expected, found, true)}" + +proc assertEq(lhs, rhs: (string, string)) = + assertEq(lhs[0], rhs[0]) + assertEq(lhs[1], rhs[1]) + +diff("a", "b").assertEq: + """ +[D/- ][R/a] +[I/+ ][R/b]""" + # `-` and `+` are formatted as delete/insert operations, `a` is formatted + # as `Replace` + +diff("a b", "b b").assertEq: + """ +[D/- ][R/a]#[K/ ]#[K/b] +[I/+ ][R/b]#[K/ ]#[K/b]""" +# `Keep` the space and last `b`, replace first `a -> b`. Space is in +# the middle of the diff, so it is not considered 'invisible' and not +# highlighted explicitly. + +diff("", "\n", true).assertEq: + """ +[K/~ ][K/] [K/~ ][K/][I/[LF]] +[N/? ] [I/+ ][I/]""" + +# Keep the empty line. `[LF]` at the end is not considered for diff +# since it used to *separate* lines, but it is annotated as a +# difference. + +diff("a ", "a").assertEq: + """ +[D/- ][K/a]#[D/[SPC]] +[I/+ ][K/a]""" +# Deleted trailing space + +# Missing leading whitespace is not considered an 'invisible' character +# for both regular and line diffs. +diff(" a", "a", true).assertEq("[D/- ][D/ ]#[K/a] [I/+ ][K/a]") +ldiff(" a", "a").assertEq(("[D/ ]#[K/a]", "[K/a]")) +# Intermediate whitespace is not invisible as well +ldiff("a a", "a").assertEq(("[D/a]#[D/ ]#[K/a]", "[K/a]")) +# Trailing whitespace IS invisible +ldiff("a ", "a").assertEq(("[K/a]#[D/[SPC]]", "[K/a]")) + +# Control characters ARE invisible, regardless of their position in the +# text, so they are explicitly shown in diffs +ldiff("\ea", "a").assertEq(("[R/[ESC]a]", "[R/a]")) + +# Inline edit diff annotations - for spelsuggest, invalid CLI switches, +# misspelled words, spell annotations, high-granularity diff suggestions. +ediff("a", "b").assertEq("[R/-]") # Replace 'a' with 'b' + +# Replace first 'a', delete second one. Edit streaks are grouped +ediff("a a", "b").assertEq("[R/-]#[D/ a]") +# Elements between blocks are joined with `#` character, just like +# regular inline diff elements +ediff("w o r d", "w e r d").assertEq("[K/w ]#[R/-]#[K/ r d]") + +conf.maxUnchanged = 2 + +diff(""" +* +* +* +* +^ +* +* +* +""", """ +* +* +* +* +& +* +* +* +""").assertEq(""" +[K/~ ][K/*] +[K/~ ][K/*] +[D/- ][R/^] +[I/+ ][R/&] +[K/~ ][K/*] +[K/~ ][K/*]""") + +# Show only two unchanged lines before/after the change + +conf.maxUnchangedWords = 1 +ldiff( + "@ @ @ @ @ @ @ @ @ @ @ @", + "@ @ @ @ @ ! @ @ @ @ @ @", +).assertEq(( + # show 'Keep' `@` and `@` for one element around the edit operations, + # discard everything else. + "[K/@]#[K/ ]#[R/@]#[K/ ]#[K/@]", + "[K/@]#[K/ ]#[R/!]#[K/ ]#[K/@]" +)) + +diff("\n", "").assertEq(""" +[K/~ ][K/][D/[LF]] +[D/- ][D/]""") +# The line itself was modified but the newline character at the end was +# removed. This change is not considered as an edit operation + +diff("", "\n", true).assertEq(""" +[K/~ ][K/] [K/~ ][K/][I/[LF]] +[N/? ] [I/+ ][I/]""") + +# Inserted newline is not a diff /directly/ as well - the *next* line +# that was modified (inserted). But new trailing newline is shown here + +if false: + # Debugging rid setup. Code needs to compile, but running not a part of + # the test. + + echo diff(""" + (User :str "User Hint" :location ("tfile.nim" 8 _))""", """ + (User :severity Hint :str "User hint" :location ("tfile_regular.nim" 8 6)) + (User :severity Hint :str "Another hint" :location ("tfile_regular.nim" 10 6))""") + + block: + for (l, r) in @[ + (""" + old text1 + """, """ + old txt1 + """), (""" + old text1 + old text2 + """, """ + old text1 + old text2 + """), (""" + (User :str "User Hint" :location ("tfile.nim" 8 _))""", """ + (User :severity Hint :str "User hint" :location ("tfile_regular.nim" 8 6)) + (User :severity Hint :str "Another hint" :location ("tfile_regular.nim" 10 6))""") + ]: + for g in [true, false]: + var fmt = diffFormatter() + fmt.groupLine = g + echo ">>>" + echo formatDiffed(l, r, fmt) + + # fmt.formatChunk = proc( + # text: string, mode, secondary: SeqEditKind, + # inline: bool + # ): ColText = + # toColText("$2" % [ + # substr($mode, 3), text + # ]) + + # echo ".. raw:: html\n" + # echo " 
"
+        # echo formatDiffed(l, r, fmt).indent(4)
+        # echo "
" diff --git a/tests/stdlib/tsexp_diff.nim b/tests/stdlib/tsexp_diff.nim new file mode 100644 index 00000000000..fe990ddd601 --- /dev/null +++ b/tests/stdlib/tsexp_diff.nim @@ -0,0 +1,242 @@ +discard """ +description: ''' +Test structural S-expression comparison. Correctness of +this test is very important, since it directly influences +testament UX when it comes to structural data comparisons. +''' +""" + +import experimental/[sexp, sexp_diff, colordiff] +import std/[tables, algorithm] + +proc sortmatches(s1, s2: seq[SexpNode], dir: SortOrder = Ascending): + tuple[ + lhsIgnore, rhsIgnore: seq[int], + map: seq[tuple[key: (int, int), diff: seq[SexpMismatch]]], + ] = + + var diffMap = TableRef[(int, int), seq[SexpMismatch]]() + + proc reportCmp(a, b: int): int = + # NOTE weight function used in the test formatting - it is based on the + # number of mismatches. Other implementations might be more + # involved/optimized, but for the testing purposes simple version will + # do. + let diff = diff(s1[a], s2[b]) + diffMap[(a, b)] = diff + return diff.len + + let (expected, given, sorted) = stablematch( + s1.len, s2.len, reportCmp, dir) + + for (key, val) in sorted: + result.map.add((key, diffMap[key])) + + result.lhsIgnore = expected + result.rhsIgnore = given + +proc sortmatchesIdx( + s1, s2: seq[string], + dir: SortOrder = Ascending + ): auto = + + var ss1, ss2: seq[SexpNode] + for item in s1: ss1.add parseSexp(item) + for item in s2: ss2.add parseSexp(item) + let (l, r, mis) = sortmatches(ss1, ss2, dir) + return (expected: l, given: r, idxs: mis, parsed: (ss1, ss2)) + +proc sortmatches( + s1, s2: seq[string], + dir: SortOrder = Ascending + ): tuple[ + expected, given: seq[int], + map: seq[tuple[key: (SexpNode, SexpNode), diff: seq[SexpMismatch]]]] = + var (expected, given, idxs, parsed) = sortmatchesIdx(s1, s2, dir) + for (idx, diff) in idxs: + result.map.add(((parsed[0][idx[0]], parsed[1][idx[1]]), diff)) + + result.expected = expected + result.given = given + +proc matches(s1, s2: string): seq[SexpMismatch] = + diff(s1.parseSexp(), s2.parseSexp()) + +proc eq[T](a, b: T) = + doAssert a == b, "a was " & $a & ", b was " & $b + +block literal: + let d = matches("(1)", "(2)") + eq d.len, 1 + eq(d[0].kind, smDifferentLiteral) + +block symbol: + let d = matches("(A)", "(B)") + eq d.len, 1 + eq d[0].kind, smDifferentSymbol + +block match_all: + let d = matches("(_)", "(Q)") + eq d.len, 0 + +block match_keys: + block ordered_unordered: + for (m1, m2) in @[ + ("(:a b :c d)", "(:a q :c z)"), + ("(:a b :c d)", "(:c q :a z)") + ]: + let d = matches(m1, m2) + eq d.len, 2 + # Mismatches are placed in the same order as they are used in the input + # text + eq d[0].path[0].key, "a" + eq d[1].path[0].key, "c" + eq d[0].kind, smDifferentSymbol + eq d[1].kind, smDifferentSymbol + + block elements: + let d = matches("(User :key 12)", "(Azer :id 14 :key 3)") + eq d.len, 2 + eq d[0].kind, smDifferentLiteral + eq d[1].kind, smDifferentSymbol + +block weighed_matching: + let d = sortmatchesIdx(@["(User)"], @["(User2)"]).idxs + eq d[0].key, (0, 0) + eq d[0].diff.len, 1 + eq d[0].diff[0].kind, smDifferentSymbol + +block hint_matching: + let d = sortmatches(@[ + """(User :location ("tfile.nim" 11 7) :severity Hint :str "Another hint")""", + """(User :location ("tfile.nim" 8 _) :str "User Hint")""" + ], @[ + """(User :location ("tfile.nim" 8 _) :str "User Hint")""", + """(User :location ("tfile.nim" 9 6) :severity Hint :str "User hint")""" + ]) + +block direct_1_1: + let d = sortmatchesIdx(@["(T)"], @["(T)"], Descending).idxs + # Structural mapping between two groups with no differences, and one + # element in each one obviously shows no differences in a single mapping pair + eq d.len, 1 + eq d[0].diff.len, 0 + + # And direct mapping between each element + eq d[0].key, (0, 0) + +block more_expected: + block descending: + let (expected, given, idxs, _) = sortmatchesIdx( + @["(T1)", "(T2)"], @["(T1)"], Descending) + + # If expected data has more elements then best mapping will be put into + # results, and all other variants will be discarded. + + eq expected, @[1] # discarding `T2` as it does not match + eq given, @[] + + eq idxs.len, 1 + eq idxs[0].diff.len, 0 + eq idxs[0].key, (0, 0) + + block: + # Note that when used in `Ascending` option, this mapping will be + # revesed - worst possible (highest diff cost) options will be + # assigned. + let (exp, give, d, _) = sortmatchesIdx( + @["(T1)", "(T2)"], @["(T1)"], Ascending) + + # Low-weight match was discarded, higher-valued one was selected + # instead. + eq exp, @[0] + eq give, @[] + + eq d.len, 1 + eq d[0].diff.len, 1 + eq d[0].key, (1, 0) + eq d[0].diff[0].kind, smDifferentSymbol + +block more_given: + let (e, g, d, _) = sortmatchesIdx(@["(T1)"], @["(T1)", "(T2)"], Descending) + + # If more input mappings are given then everything that was not matched + # is discarded + eq e, @[] + eq g, @[1] + eq d.len, 1 + eq d[0].diff.len, 0 + +block: + # In case of multiple possible pairings the matching heavily depends on + # the ordering option. In case of Ascending it will generate the highest + # overall cost, and for descending it will make a lowest overall cost. + let (s1, s2) = (@["(A B C R)", "(A Q D R)"], + @["(A B C E)", "(A Q D E)"]) + + block ascending: + let (e, g, d, _) = sortmatchesIdx(s1, s2, Ascending) + + eq e, @[] + eq g, @[] + + eq d.len, 2 + # Overall cost - 6, best possible matching for Ascending ordering + eq d[0].diff.len, 3 + eq d[1].diff.len, 3 + + let (d1, d2, d3) = (d[0].diff[0], d[0].diff[1], d[0].diff[2]) + + eq d[0].key, (0, 1) + eq d[1].key, (1, 0) + + eq d1.path[0].index, 1 + eq d1.kind, smDifferentSymbol + eq d1.expected.getSymbol(), "B" + eq d1.found.getSymbol(), "Q" + + eq d2.path[0].index, 2 + eq d2.kind, smDifferentSymbol + + eq d3.path[0].index, 3 + eq d3.kind, smDifferentSymbol + + block descending: + let (e, g, d, _) = sortmatchesIdx(s1, s2, Descending) + + eq e, @[] + eq g, @[] + + eq d.len, 2 + # Overall cost - 2, best possible matching for Descending ordering + eq d[0].diff.len, 1 + eq d[1].diff.len, 1 + + + eq d[0].key, (0, 0) + eq d[1].key, (1, 1) + + let d1 = (d[0].diff[0]) + eq d1.path[0].index, 3 + eq d1.kind, smDifferentSymbol + eq d1.expected.getSymbol(), "R" + eq d1.found.getSymbol(), "E" + +if false: + # Don't delete this section, it is used for print-debugging expected + # formatting. And yes, 'if' is intentional as well - code needs to + # compile, running is optional. + for (lhs, rhs) in @[ + (@["(A B C R)", "(A Q D R)"], + @["(A B C E)", "(A Q D E)"]) + ]: + for dir in [Ascending, Descending]: + echo ">>>" + let (expected, given, map) = sortmatches(lhs, rhs, dir) + for (pair, diff) in map: + echo "-- ", pair[0] + echo "++ ", pair[1] + echo describeDiff(diff, diffFormatter()) + + echo "-? ", expected + echo "+? ", given diff --git a/tests/testament/tspecialpaths.nim b/tests/testament/tspecialpaths.nim index 3c97dc88adc..198d5ff8a3f 100644 --- a/tests/testament/tspecialpaths.nim +++ b/tests/testament/tspecialpaths.nim @@ -5,5 +5,3 @@ block: # splitTestFile doAssert splitTestFile("/pathto/tests/fakedir/tfakename.nim") == ("fakedir", "/pathto/tests/fakedir/tfakename.nim".unixToNativePath) doAssert splitTestFile(getCurrentDir() / "tests/fakedir/tfakename.nim") == ("fakedir", "tests/fakedir/tfakename.nim".unixToNativePath) doAssert splitTestFile(getCurrentDir() / "sub/tests/fakedir/tfakename.nim") == ("fakedir", "sub/tests/fakedir/tfakename.nim".unixToNativePath) - doAssertRaises(AssertionDefect): discard splitTestFile("testsbad/fakedir/tfakename.nim") - doAssertRaises(AssertionDefect): discard splitTestFile("tests/tfakename.nim") diff --git a/tools/koch/kochdocs.nim b/tools/koch/kochdocs.nim index 70570134aa2..62f0391b56f 100644 --- a/tools/koch/kochdocs.nim +++ b/tools/koch/kochdocs.nim @@ -323,7 +323,6 @@ lib/system/widestrs.nim a.nativeToUnixPath in docIgnore: continue result.add a - result.add normalizePath("nimsuggest/sexp.nim") let doc = getDocList()