feat(test): report skipped tests

[onelson]

Fixes #4576

While looping over the discovered tests, mark them as skipped if they
are listed on the CLI as such.

Later, while summarizing the test outcome, report the counts for the total,
failures and skipped tests.

---

Since the change was to the test runner itself, I wasn't sure of the best verification strategy. I setup a manual test using the following method.

First, I created an isolated test root with the following file:

$ cat test-root/reporting_test.flux 
package skipping

import "array"
import "testing"


input = () => array.from(rows: [{x: 1}])

testcase works {
    testing.diff(want: input(), got: input()) |> yield()
}


testcase breaks {
    want = input()
    got = input() |> filter(fn: (r) => false)
    testing.diff(want: want, got: got) |> yield()
}

The idea being it contains one test that should pass, and one that should fail.

Next, I ran the test runner on this and checked the output while specifying each of the two tests is skipped separately, and finally both together.

$ go run ./cmd/flux test -p test-root/ --skip works
sx
failures:

        breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


---
Found 2 tests: passed 0, failed 1, skipped 1
#  _diff=string,x=int
_diff=-,x=1i

exit status 1

$ go run ./cmd/flux test -p test-root/ --skip breaks
.s
---
Found 2 tests: passed 1, failed 0, skipped 1

$ go run ./cmd/flux test -p test-root/ --skip breaks,works
ss
---
Found 2 tests: passed 0, failed 0, skipped 2

Finally, I repeated this step for -v and -vv to verify the higher verbosity levels still reported the expected outcomes.

Click to see the full output.

Using -v:

$ go run ./cmd/flux test -p test-root/ --skip breaks -v
works...success
breaks...skip

---
Found 2 tests: passed 1, failed 0, skipped 1

$ go run ./cmd/flux test -p test-root/ --skip works -v
works...skip
breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


failures:

        breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


---
Found 2 tests: passed 0, failed 1, skipped 1
#  _diff=string,x=int
_diff=-,x=1i

exit status 1

$ go run ./cmd/flux test -p test-root/ --skip breaks,works -v
works...skip
breaks...skip

---
Found 2 tests: passed 0, failed 0, skipped 2

Using -vv:

$ go run ./cmd/flux test -p test-root/ --skip breaks -vv
Full source for test case "works"
// File: test-root/reporting_test.flux
package main


import "array"
import "testing"

input = () => array.from(rows: [{x: 1}])

testing.diff(want: input(), got: input()) |> yield()
works...success
Full source for test case "breaks"
// File: test-root/reporting_test.flux
package main


import "array"
import "testing"

input = () => array.from(rows: [{x: 1}])

want = input()
got = input() |> filter(fn: (r) => false)

testing.diff(want: want, got: got) |> yield()
breaks...skip

---
Found 2 tests: passed 1, failed 0, skipped 1

$ go run ./cmd/flux test -p test-root/ --skip works -vv
Full source for test case "works"
// File: test-root/reporting_test.flux
package main


import "array"
import "testing"

input = () => array.from(rows: [{x: 1}])

testing.diff(want: input(), got: input()) |> yield()
works...skip
Full source for test case "breaks"
// File: test-root/reporting_test.flux
package main


import "array"
import "testing"

input = () => array.from(rows: [{x: 1}])

want = input()
got = input() |> filter(fn: (r) => false)

testing.diff(want: want, got: got) |> yield()
breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


failures:

        breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


---
Found 2 tests: passed 0, failed 1, skipped 1
#  _diff=string,x=int
_diff=-,x=1i

exit status 1


$ go run ./cmd/flux test -p test-root/ --skip breaks,works -vv
Full source for test case "works"
// File: test-root/reporting_test.flux
package main


import "array"
import "testing"

input = () => array.from(rows: [{x: 1}])

testing.diff(want: input(), got: input()) |> yield()
works...skip
Full source for test case "breaks"
// File: test-root/reporting_test.flux
package main


import "array"
import "testing"

input = () => array.from(rows: [{x: 1}])

want = input()
got = input() |> filter(fn: (r) => false)

testing.diff(want: want, got: got) |> yield()
breaks...skip

---
Found 2 tests: passed 0, failed 0, skipped 2

Done checklist

• docs/SPEC.md updated N/A
• Test cases written

[onelson]

I just now realized I never ran this without any skips specified, so here's the output from that final case:

$ go run ./cmd/flux test -p test-root/ 
.x
failures:

        breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


---
Found 2 tests: passed 1, failed 1, skipped 0
#  _diff=string,x=int
_diff=-,x=1i

exit status 1


$ go run ./cmd/flux test -p test-root/ -v
works...success
breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


failures:

        breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


---
Found 2 tests: passed 1, failed 1, skipped 0
#  _diff=string,x=int
_diff=-,x=1i

exit status 1


$ go run ./cmd/flux test -p test-root/ -vv
Full source for test case "works"
// File: test-root/reporting_test.flux
package main


import "array"
import "testing"

input = () => array.from(rows: [{x: 1}])

testing.diff(want: input(), got: input()) |> yield()
works...success
Full source for test case "breaks"
// File: test-root/reporting_test.flux
package main


import "array"
import "testing"

input = () => array.from(rows: [{x: 1}])

want = input()
got = input() |> filter(fn: (r) => false)

testing.diff(want: want, got: got) |> yield()
breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


failures:

        breaks...fail: #  _diff=string,x=int
_diff=-,x=1i


---
Found 2 tests: passed 1, failed 1, skipped 0
#  _diff=string,x=int
_diff=-,x=1i

exit status 1

[onelson]

Additional thought, while we're in here.

We might change the fail output to be f instead of x, since that would open the door to later adopting f712u compatible output.