Skip to content

Commit

Permalink
testament megatest: we can now tell which test failed; helps debuggin…
Browse files Browse the repository at this point in the history
…g and prevents certain bugs, plus other fixes (nim-lang#10089)

 * [testament] --verboseMegatest flag to make megatest compilation verbose
* replace "tests" by testsDir
* megatest's nimcache is now in same dir as other tests to avoid clobbering (eg when running tests from multiple Nim repos)
  • Loading branch information
timotheecour authored and Araq committed Jan 5, 2019
1 parent ffea3fb commit 4a72039
Show file tree
Hide file tree
Showing 3 changed files with 50 additions and 20 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,7 @@ test.txt
tweeter.db
tweeter_test.db
megatest.nim

/outputExpected.txt
/outputGotten.txt

57 changes: 40 additions & 17 deletions testament/categories.nim
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ proc compileRodFiles(r: var TResults, cat: Category, options: string) =

proc flagTests(r: var TResults, cat: Category, options: string) =
# --genscript
const filename = "tests"/"flags"/"tgenscript"
const filename = testsDir/"flags"/"tgenscript"
const genopts = " --genscript"
let nimcache = nimcacheDir(filename, genopts, targetC)
testSpec r, makeTest(filename, genopts, cat)
Expand Down Expand Up @@ -356,7 +356,7 @@ proc testNimInAction(r: var TResults, cat: Category, options: string) =
]

for i, test in tests:
let filename = "tests" / test.addFileExt("nim")
let filename = testsDir / test.addFileExt("nim")
let testHash = getMD5(readFile(filename).string)
doAssert testHash == refHashes[i], "Nim in Action test " & filename & " was changed."
# Run the tests.
Expand Down Expand Up @@ -425,15 +425,12 @@ proc testStdlib(r: var TResults, pattern, options: string, cat: Category) =
let contents = readFile(testFile).string
var testObj = makeTest(testFile, options, cat)
#[
TODO:
todo:
this logic is fragile:
false positives (if appears in a comment), or false negatives, eg
`when defined(osx) and isMainModule`.
Instead of fixing this, a much better way, is to extend
https://github.com/nim-lang/Nim/issues/9581 to stdlib modules as follows:
* add these to megatest
* patch compiler so `isMainModule` is true when -d:isMainModuleIsAlwaysTrue
That'll give speedup benefit, and we don't have to patch stdlib files.
Instead of fixing this, see https://github.com/nim-lang/Nim/issues/10045
for a much better way.
]#
if "when isMainModule" notin contents:
testObj.spec.action = actionCompile
Expand Down Expand Up @@ -534,7 +531,7 @@ proc `&.?`(a, b: string): string =
result = if b.startswith(a): b else: a & b

proc processSingleTest(r: var TResults, cat: Category, options, test: string) =
let test = "tests" & DirSep &.? cat.string / test
let test = testsDir &.? cat.string / test
let target = if cat.string.normalize == "js": targetJS else: targetC
if existsFile(test):
testSpec r, makeTest(test, options, cat), {target}
Expand All @@ -559,16 +556,21 @@ proc isJoinableSpec(spec: TSpec): bool =
(spec.targets == {} or spec.targets == {targetC})

proc norm(s: var string) =
# equivalent of s/\n+/\n/g (could use a single pass over input if needed)
while true:
let tmp = s.replace("\n\n", "\n")
if tmp == s: break
s = tmp
s = s.strip

proc isTestFile*(file: string): bool =
let (dir, name, ext) = splitFile(file)
let (_, name, ext) = splitFile(file)
result = ext == ".nim" and name.startsWith("t")

proc quoted(a: string): string =
# todo: consider moving to system.nim
result.addQuoted(a)

proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
## returs a list of tests that have problems
var specs: seq[TSpec] = @[]
Expand All @@ -582,6 +584,8 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
if isJoinableSpec(spec):
specs.add spec

proc cmp(a: TSpec, b:TSpec): auto = cmp(a.file, b.file)
sort(specs, cmp=cmp) # reproducible order
echo "joinable specs: ", specs.len

if simulate:
Expand All @@ -591,19 +595,36 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
return

var megatest: string
for runSpec in specs:
megatest.add "import r\""
megatest.add runSpec.file
megatest.add "\"\n"
#[
TODO(minor):
get from Nim cmd
put outputGotten.txt, outputGotten.txt, megatest.nim there too
delete upon completion, maybe
]#
var outDir = nimcacheDir(testsDir / "megatest", "", targetC)
const marker = "megatest:processing: "

for i, runSpec in specs:
let file = runSpec.file
let file2 = outDir / ("megatest_" & $i & ".nim")
# `include` didn't work with `trecmod2.nim`, so using `import`
let code = "echo \"" & marker & "\", " & quoted(file) & "\n"
createDir(file2.parentDir)
writeFile(file2, code)
megatest.add "import " & quoted(file2) & "\n"
megatest.add "import " & quoted(file) & "\n"

writeFile("megatest.nim", megatest)

const args = ["c", "-d:testing", "--listCmd", "megatest.nim"]
var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "")
let args = ["c", "--nimCache:" & outDir, "-d:testing", "--listCmd", "megatest.nim"]
proc onStdout(line: string) = echo line
var (buf, exitCode) = execCmdEx2(command = compilerPrefix, args = args, options = {poStdErrToStdOut, poUsePath}, input = "",
onStdout = if verboseMegatest: onStdout else: nil)
if exitCode != 0:
echo buf
quit("megatest compilation failed")

# Could also use onStdout here.
(buf, exitCode) = execCmdEx("./megatest")
if exitCode != 0:
echo buf
Expand All @@ -613,6 +634,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
writeFile("outputGotten.txt", buf)
var outputExpected = ""
for i, runSpec in specs:
outputExpected.add marker & runSpec.file & "\n"
outputExpected.add runSpec.output.strip
outputExpected.add '\n'
norm outputExpected
Expand All @@ -621,6 +643,7 @@ proc runJoinedTest(r: var TResults, cat: Category, testsDir: string) =
writeFile("outputExpected.txt", outputExpected)
discard execShellCmd("diff -uNdr outputExpected.txt outputGotten.txt")
echo "output different!"
# outputGotten.txt, outputExpected.txt not removed on purpose for debugging.
quit 1
else:
echo "output OK"
Expand Down Expand Up @@ -686,7 +709,7 @@ proc processCategory(r: var TResults, cat: Category, options, testsDir: string,
var testsRun = 0

var files: seq[string]
for file in walkDirRec("tests" & DirSep &.? cat.string):
for file in walkDirRec(testsDir &.? cat.string):
if isTestFile(file): files.add file
files.sort # give reproducible order

Expand Down
9 changes: 6 additions & 3 deletions testament/tester.nim
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ import
var useColors = true
var backendLogging = true
var simulate = false
var verboseMegatest = false # very verbose but can be useful

const
testsDir = "tests" & DirSep
resultsFile = "testresults.html"
#jsonFile = "testresults.json" # not used
Usage = """Usage:
Expand All @@ -34,6 +36,7 @@ Arguments:
arguments are passed to the compiler
Options:
--print also print results to the console
--verboseMegatest log to stdout megatetest compilation
--simulate see what tests would be run but don't run them (for debugging)
--failing only show failing/ignored tests
--targets:"c c++ js objc" run tests for specified targets (default: all)
Expand Down Expand Up @@ -83,7 +86,7 @@ proc getFileDir(filename: string): string =
if not result.isAbsolute():
result = getCurrentDir() / result

proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOption], input: string): tuple[
proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOption], input: string, onStdout: proc(line: string) = nil): tuple[
output: TaintedString,
exitCode: int] {.tags:
[ExecIOEffect, ReadIOEffect, RootEffect], gcsafe.} =
Expand All @@ -103,6 +106,7 @@ proc execCmdEx2(command: string, args: openarray[string], options: set[ProcessOp
if outp.readLine(line):
result[0].string.add(line.string)
result[0].string.add("\n")
if onStdout != nil: onStdout(line.string)
else:
result[1] = peekExitCode(p)
if result[1] != -1: break
Expand Down Expand Up @@ -517,8 +521,6 @@ else:

include categories

const testsDir = "tests" & DirSep

proc main() =
os.putenv "NIMTEST_COLOR", "never"
os.putenv "NIMTEST_OUTPUT_LVL", "PRINT_FAILURES"
Expand All @@ -533,6 +535,7 @@ proc main() =
while p.kind == cmdLongoption:
case p.key.string.normalize
of "print", "verbose": optPrintResults = true
of "verbosemegatest": verboseMegatest = true
of "failing": optFailing = true
of "pedantic": discard "now always enabled"
of "targets":
Expand Down

0 comments on commit 4a72039

Please sign in to comment.