diff --git a/docs/getting-started.md b/docs/getting-started.md index a743d0ff..d59029cc 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -124,12 +124,13 @@ MUnit test suites can be executed from VS Code like normal test suites. Test results are formatted in a specific way to make it easy to search for them in a large log file. -| Test | Prefix | -| ------- | ------- | -| Success | `+` | -| Failed | `==> X` | -| Ignored | `==> i` | -| Skipped | `==> s` | +| Test | Prefix | Comment | See Also | +| ------- | ------- | --------- | ------------------------------------- | +| Success | `+` | | | +| Failed | `==> X` | | [Writing assertions](assertions.html) | +| Ignored | `==> i` | `ignored` | [Filtering tests](filtering.html) | +| Pending | `==> i` | `PENDING` | [Declaring tests](tests.html) | +| Skipped | `==> s` | | [Filtering tests](filtering.html) | Knowing these prefixes may come in handy for example when browsing test logs in a browser. Search for `==> X` to quickly navigate to the failed tests. diff --git a/docs/scalatest.md b/docs/scalatest.md index 47f327c5..61ee6feb 100644 --- a/docs/scalatest.md +++ b/docs/scalatest.md @@ -41,6 +41,11 @@ If you only use basic ScalaTest features, you should be able to replace usage of + test("ignored".ignore) { // unchanged } + +- test("pending") (pending) ++ test("pending".pending) { ++ // zero or more assertions ++ } ``` If you are coming from `WordSpec` style tests, make sure to flatten them, or your tests diff --git a/docs/tests.md b/docs/tests.md index ccbc5e2b..3228ab37 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -356,6 +356,55 @@ what Scala version caused the tests to fail. + munit.ScalaVersionFrameworkSuite.foo-2.13.1 ``` +## Tag pending tests + +Use `.pending` to annotate a work-in-progress test case with known-incomplete coverage. +Any assertions in the pending case must pass, but will be reported as Ignored instead of Success. +Any failures will be reported as Failures (this differs from `.ignore` which skips the test case entirely). +This tag is useful for documenting: + +- **Empty placeholders** that lack any assertions (unless tagged pending, these are reported as success). +- **Incomplete placeholders** with too few assertions (unless tagged pending, these are reported as success). +- **Accurate placeholders** whose stable assertions must pass (regressions are not ignored). +- **Searchability** of your codebase for known-incomplete test cases. +- **Cross-references** between your codebase and issue trackers. + +You can (optionally) include a comment for your pending test case, such as a job ticket ID: + +```scala + // Empty placeholder, without logged comments: + test("time travel".pending) { + // Test case to be written yesterday + } + + // Empty placeholder, with logged comments: + test("time travel".pending("requirements from product owner")) { + // Is this funded yet?? + } + + // Empty placeholder, tracked for action: + test("time travel".pending("INTERN-101")) { + // Test case to be written by an intern + } + + // Incomplete (WIP) placeholder, tracked for action: + test("time travel".pending("QA-404")) { + assert(LocalDate.now.isAfter(yesterday)) + // QA team to provide specific examples for regression-test coverage + } +``` + +If you want to mark a failed regression test as pending-until-fixed, +you combine `.ignore` before or after `.pending`, for example: + +```scala + test("this test worked yesterday".ignore.pending("platform investigation")) { + assert(LocalDate.now.equals(yesterday)) + } +``` + +This allows pending comments, reasons, or cross-references to be logged for ignored tests. + ## Tag flaky tests Use `.flaky` to mark a test case that has a tendency to non-deterministically diff --git a/junit-interface/src/main/java/munit/internal/junitinterface/EventDispatcher.java b/junit-interface/src/main/java/munit/internal/junitinterface/EventDispatcher.java index 70e7014d..b0f6136f 100644 --- a/junit-interface/src/main/java/munit/internal/junitinterface/EventDispatcher.java +++ b/junit-interface/src/main/java/munit/internal/junitinterface/EventDispatcher.java @@ -2,6 +2,7 @@ import static munit.internal.junitinterface.Ansi.*; +import java.lang.annotation.Annotation; import java.util.Arrays; import java.util.Collections; import java.util.Set; @@ -133,10 +134,28 @@ public void testIgnored(Description desc) { desc, new InfoEvent(desc, Status.Ignored) { void logTo(RichLogger logger) { + StringBuilder builder = new StringBuilder(1024); + boolean isPending = false; + for (Annotation annotation : desc.getAnnotations()) { + if (annotation instanceof Tag) { + Tag tag = (Tag) annotation; + if (tag instanceof PendingTag) { + isPending = true; + } + else if (tag instanceof PendingCommentTag) { + builder.append(" "); + builder.append(tag.value()); + } + } + } + if (isPending) { + builder.insert(0, " PENDING"); + } + builder.append(" ignored"); logger.warn( settings.buildTestResult(Status.Ignored) + ansiName - + Ansi.c(" ignored", SKIPPED) + + Ansi.c(builder.toString(), SKIPPED) + durationSuffix()); } }); diff --git a/junit-interface/src/main/java/munit/internal/junitinterface/PendingCommentTag.java b/junit-interface/src/main/java/munit/internal/junitinterface/PendingCommentTag.java new file mode 100644 index 00000000..27eb7392 --- /dev/null +++ b/junit-interface/src/main/java/munit/internal/junitinterface/PendingCommentTag.java @@ -0,0 +1,3 @@ +package munit.internal.junitinterface; + +public interface PendingCommentTag extends Tag {} diff --git a/junit-interface/src/main/java/munit/internal/junitinterface/PendingTag.java b/junit-interface/src/main/java/munit/internal/junitinterface/PendingTag.java new file mode 100644 index 00000000..a864caf8 --- /dev/null +++ b/junit-interface/src/main/java/munit/internal/junitinterface/PendingTag.java @@ -0,0 +1,3 @@ +package munit.internal.junitinterface; + +public interface PendingTag extends Tag {} diff --git a/munit/js-native/src/main/scala/munit/internal/junitinterface/JUnitReporter.scala b/munit/js-native/src/main/scala/munit/internal/junitinterface/JUnitReporter.scala index 5837607d..4669faa2 100644 --- a/munit/js-native/src/main/scala/munit/internal/junitinterface/JUnitReporter.scala +++ b/munit/js-native/src/main/scala/munit/internal/junitinterface/JUnitReporter.scala @@ -29,8 +29,17 @@ final class JUnitReporter( } } - def reportTestIgnored(method: String): Unit = { - log(Info, AnsiColors.c(s"==> i $method ignored", AnsiColors.YELLOW)) + def reportTestIgnored( + method: String, + elapsedMillis: Double, + suffix: String + ): Unit = { + val suffixed = if (suffix.isEmpty) "" else s" ${suffix}" + log( + Info, + AnsiColors.c(s"==> i $method$suffixed ignored", AnsiColors.YELLOW) + " " + + formatTime(elapsedMillis) + ) emitEvent(method, Status.Ignored) } def reportAssumptionViolation( diff --git a/munit/js-native/src/main/scala/munit/internal/junitinterface/MUnitRunNotifier.scala b/munit/js-native/src/main/scala/munit/internal/junitinterface/MUnitRunNotifier.scala index b03af1ff..ec4092dc 100644 --- a/munit/js-native/src/main/scala/munit/internal/junitinterface/MUnitRunNotifier.scala +++ b/munit/js-native/src/main/scala/munit/internal/junitinterface/MUnitRunNotifier.scala @@ -24,7 +24,21 @@ class MUnitRunNotifier(reporter: JUnitReporter) extends RunNotifier { override def fireTestIgnored(description: Description): Unit = { ignored += 1 isReported += description - reporter.reportTestIgnored(description.getMethodName) + val pendingSuffixes = { + val annotations = description.getAnnotations + val isPending = annotations.collect { case munit.Pending => + "PENDING" + }.distinct + val pendingComments = annotations.collect { + case tag: munit.PendingComment => tag.value + } + isPending ++ pendingComments + } + reporter.reportTestIgnored( + description.getMethodName, + elapsedMillis(), + pendingSuffixes.mkString(" ") + ) } override def fireTestAssumptionFailed( failure: notification.Failure diff --git a/munit/js-native/src/main/scala/munit/internal/junitinterface/PendingCommentTag.scala b/munit/js-native/src/main/scala/munit/internal/junitinterface/PendingCommentTag.scala new file mode 100644 index 00000000..c01510a1 --- /dev/null +++ b/munit/js-native/src/main/scala/munit/internal/junitinterface/PendingCommentTag.scala @@ -0,0 +1,3 @@ +package munit.internal.junitinterface + +trait PendingCommentTag extends Tag diff --git a/munit/js-native/src/main/scala/munit/internal/junitinterface/PendingTag.scala b/munit/js-native/src/main/scala/munit/internal/junitinterface/PendingTag.scala new file mode 100644 index 00000000..29bde139 --- /dev/null +++ b/munit/js-native/src/main/scala/munit/internal/junitinterface/PendingTag.scala @@ -0,0 +1,3 @@ +package munit.internal.junitinterface + +trait PendingTag extends Tag diff --git a/munit/shared/src/main/scala/munit/MUnitRunner.scala b/munit/shared/src/main/scala/munit/MUnitRunner.scala index eac8d63b..9f5a3919 100644 --- a/munit/shared/src/main/scala/munit/MUnitRunner.scala +++ b/munit/shared/src/main/scala/munit/MUnitRunner.scala @@ -325,7 +325,7 @@ class MUnitRunner(val cls: Class[_ <: Suite], newInstance: () => Suite) catch onError result.map { _ => notifier.fireTestFinished(description) - true + !test.tags(Pending) } } @@ -361,6 +361,8 @@ class MUnitRunner(val cls: Class[_ <: Suite], newInstance: () => Suite) notifier.fireTestAssumptionFailed(new Failure(description, f)) case TestValues.Ignore => notifier.fireTestIgnored(description) + case _ if test.tags(Pending) => + notifier.fireTestIgnored(description) case _ => () } diff --git a/munit/shared/src/main/scala/munit/TestOptions.scala b/munit/shared/src/main/scala/munit/TestOptions.scala index 6b958133..a9805fa5 100644 --- a/munit/shared/src/main/scala/munit/TestOptions.scala +++ b/munit/shared/src/main/scala/munit/TestOptions.scala @@ -25,6 +25,9 @@ final class TestOptions( def fail: TestOptions = tag(Fail) def flaky: TestOptions = tag(Flaky) def ignore: TestOptions = tag(Ignore) + def pending: TestOptions = tag(Pending) + def pending(comment: String): TestOptions = + pending.tag(PendingComment(comment)) def only: TestOptions = tag(Only) def tag(t: Tag): TestOptions = copy(tags = tags + t) private[this] def copy( diff --git a/munit/shared/src/main/scala/munit/TestValues.scala b/munit/shared/src/main/scala/munit/TestValues.scala index f0367ab9..e8a5d419 100644 --- a/munit/shared/src/main/scala/munit/TestValues.scala +++ b/munit/shared/src/main/scala/munit/TestValues.scala @@ -17,6 +17,6 @@ object TestValues { with NoStackTrace with Serializable - /** The test case was ignored. */ + /** The test case was ignored (skipped). */ val Ignore = munit.Ignore } diff --git a/munit/shared/src/main/scala/munit/package.scala b/munit/shared/src/main/scala/munit/package.scala index 3f25dc83..fc4be115 100644 --- a/munit/shared/src/main/scala/munit/package.scala +++ b/munit/shared/src/main/scala/munit/package.scala @@ -1,7 +1,14 @@ +import munit.internal.junitinterface.{PendingCommentTag, PendingTag} + package object munit { + case class PendingComment(override val value: String) + extends Tag(value) + with PendingCommentTag + val Ignore = new Tag("Ignore") val Only = new Tag("Only") val Flaky = new Tag("Flaky") val Fail = new Tag("Fail") + val Pending: Tag with PendingTag = new Tag("Pending") with PendingTag val Slow = new Tag("Slow") } diff --git a/tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala b/tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala index 53e60ea2..9c336b3d 100644 --- a/tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala +++ b/tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala @@ -1,5 +1,7 @@ package munit +import munit.internal.io.PlatformIO.File + class SkippedFrameworkSuite extends FunSuite { test("pass") { // println("pass") @@ -7,6 +9,12 @@ class SkippedFrameworkSuite extends FunSuite { test("ignore".ignore) { ??? } + test("ignore.failed.pending".ignore.pending) { + assert(false) + } + test("ignore.failed.pending.comment".ignore.pending("comment")) { + assert(false) + } test("assume(true)") { assume(true, "assume it passes") // println("pass") @@ -14,6 +22,30 @@ class SkippedFrameworkSuite extends FunSuite { test("assume(false)") { assume(false, "assume it fails") } + test("pending.empty.ignored".pending) { + // + } + test("pending.empty.ignored.comment".pending("comment")) { + // + } + test("pending.successful.ignored".pending) { + assert(true) + } + test("pending.successful.ignored.comment".pending("comment")) { + assert(true) + } + test("pending.failed.not-ignored".pending) { + assert(false) + } + test("pending.failed.not-ignored.comment".pending("comment")) { + assert(false) + } + test("pending.failed.ignored".pending.ignore) { + assert(false) + } + test("pending.failed.ignored.comment".pending("comment").ignore) { + assert(false) + } } object SkippedFrameworkSuite @@ -21,8 +53,24 @@ object SkippedFrameworkSuite classOf[SkippedFrameworkSuite], """|==> success munit.SkippedFrameworkSuite.pass |==> ignored munit.SkippedFrameworkSuite.ignore + |==> ignored munit.SkippedFrameworkSuite.ignore.failed.pending + |==> ignored munit.SkippedFrameworkSuite.ignore.failed.pending.comment |==> success munit.SkippedFrameworkSuite.assume(true) |==> skipped munit.SkippedFrameworkSuite.assume(false) - assume it fails + |==> ignored munit.SkippedFrameworkSuite.pending.empty.ignored + |==> ignored munit.SkippedFrameworkSuite.pending.empty.ignored.comment + |==> ignored munit.SkippedFrameworkSuite.pending.successful.ignored + |==> ignored munit.SkippedFrameworkSuite.pending.successful.ignored.comment + |==> failure munit.SkippedFrameworkSuite.pending.failed.not-ignored - tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:38 assertion failed + |37: test("pending.failed.not-ignored".pending) { + |38: assert(false) + |39: } + |==> failure munit.SkippedFrameworkSuite.pending.failed.not-ignored.comment - tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:41 assertion failed + |40: test("pending.failed.not-ignored.comment".pending("comment")) { + |41: assert(false) + |42: } + |==> ignored munit.SkippedFrameworkSuite.pending.failed.ignored + |==> ignored munit.SkippedFrameworkSuite.pending.failed.ignored.comment |""".stripMargin, format = SbtFormat ) @@ -32,10 +80,26 @@ object SkippedFrameworkStdoutJsNativeSuite classOf[SkippedFrameworkSuite], """|munit.SkippedFrameworkSuite: | + pass - |==> i ignore ignored + |==> i ignore ignored + |==> i ignore.failed.pending PENDING ignored + |==> i ignore.failed.pending.comment PENDING comment ignored | + assume(true) |==> s assume(false) skipped - |""".stripMargin, + |==> i pending.empty.ignored PENDING ignored + |==> i pending.empty.ignored.comment PENDING comment ignored + |==> i pending.successful.ignored PENDING ignored + |==> i pending.successful.ignored.comment PENDING comment ignored + |==> X munit.SkippedFrameworkSuite.pending.failed.not-ignored munit.FailException: tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:38 assertion failed + |37: test("pending.failed.not-ignored".pending) { + |38: assert(false) + |39: } + |==> X munit.SkippedFrameworkSuite.pending.failed.not-ignored.comment munit.FailException: tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:41 assertion failed + |40: test("pending.failed.not-ignored.comment".pending("comment")) { + |41: assert(false) + |42: } + |==> i pending.failed.ignored PENDING ignored + |==> i pending.failed.ignored.comment PENDING comment ignored + |""".stripMargin.replace('/', File.separatorChar), format = StdoutFormat, tags = Set(NoJVM) ) @@ -46,12 +110,34 @@ object SkippedFrameworkStdoutJsNativeVerboseSuite """|munit.SkippedFrameworkSuite: |pass started | + pass - |==> i ignore ignored + |==> i ignore ignored + |==> i ignore.failed.pending PENDING ignored + |==> i ignore.failed.pending.comment PENDING comment ignored |assume(true) started | + assume(true) |assume(false) started |==> s assume(false) skipped - |""".stripMargin, + |pending.empty.ignored started + |==> i pending.empty.ignored PENDING ignored + |pending.empty.ignored.comment started + |==> i pending.empty.ignored.comment PENDING comment ignored + |pending.successful.ignored started + |==> i pending.successful.ignored PENDING ignored + |pending.successful.ignored.comment started + |==> i pending.successful.ignored.comment PENDING comment ignored + |pending.failed.not-ignored started + |==> X munit.SkippedFrameworkSuite.pending.failed.not-ignored munit.FailException: tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:38 assertion failed + |37: test("pending.failed.not-ignored".pending) { + |38: assert(false) + |39: } + |pending.failed.not-ignored.comment started + |==> X munit.SkippedFrameworkSuite.pending.failed.not-ignored.comment munit.FailException: tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:41 assertion failed + |40: test("pending.failed.not-ignored.comment".pending("comment")) { + |41: assert(false) + |42: } + |==> i pending.failed.ignored PENDING ignored + |==> i pending.failed.ignored.comment PENDING comment ignored + |""".stripMargin.replace('/', File.separatorChar), format = StdoutFormat, tags = Set(NoJVM), arguments = Array("-v") @@ -63,9 +149,29 @@ object SkippedFrameworkStdoutJVMSuite """|munit.SkippedFrameworkSuite: | + pass |==> i munit.SkippedFrameworkSuite.ignore ignored + |==> i munit.SkippedFrameworkSuite.ignore.failed.pending PENDING ignored + |==> i munit.SkippedFrameworkSuite.ignore.failed.pending.comment PENDING comment ignored | + assume(true) |==> s munit.SkippedFrameworkSuite.assume(false) skipped - |""".stripMargin, + |==> i munit.SkippedFrameworkSuite.pending.empty.ignored PENDING ignored + |==> i munit.SkippedFrameworkSuite.pending.empty.ignored.comment PENDING comment ignored + |==> i munit.SkippedFrameworkSuite.pending.successful.ignored PENDING ignored + |==> i munit.SkippedFrameworkSuite.pending.successful.ignored.comment PENDING comment ignored + |==> X munit.SkippedFrameworkSuite.pending.failed.not-ignored munit.FailException: tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:38 assertion failed + |37: test("pending.failed.not-ignored".pending) { + |38: assert(false) + |39: } + | at munit.FunSuite.assert(FunSuite.scala:11) + | at munit.SkippedFrameworkSuite.$anonfun$new$21(SkippedFrameworkSuite.scala:38) + |==> X munit.SkippedFrameworkSuite.pending.failed.not-ignored.comment munit.FailException: tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:41 assertion failed + |40: test("pending.failed.not-ignored.comment".pending("comment")) { + |41: assert(false) + |42: } + | at munit.FunSuite.assert(FunSuite.scala:11) + | at munit.SkippedFrameworkSuite.$anonfun$new$24(SkippedFrameworkSuite.scala:41) + |==> i munit.SkippedFrameworkSuite.pending.failed.ignored PENDING ignored + |==> i munit.SkippedFrameworkSuite.pending.failed.ignored.comment PENDING comment ignored + |""".stripMargin.replace('/', File.separatorChar), format = StdoutFormat, tags = Set(OnlyJVM) ) @@ -78,12 +184,38 @@ object SkippedFrameworkStdoutJVMVerboseSuite |munit.SkippedFrameworkSuite.pass started | + pass |==> i munit.SkippedFrameworkSuite.ignore ignored + |==> i munit.SkippedFrameworkSuite.ignore.failed.pending PENDING ignored + |==> i munit.SkippedFrameworkSuite.ignore.failed.pending.comment PENDING comment ignored |munit.SkippedFrameworkSuite.assume(true) started | + assume(true) |munit.SkippedFrameworkSuite.assume(false) started |==> s munit.SkippedFrameworkSuite.assume(false) skipped - |Test run munit.SkippedFrameworkSuite finished: 0 failed, 1 ignored, 3 total, - |""".stripMargin, + |munit.SkippedFrameworkSuite.pending.empty.ignored started + |==> i munit.SkippedFrameworkSuite.pending.empty.ignored PENDING ignored + |munit.SkippedFrameworkSuite.pending.empty.ignored.comment started + |==> i munit.SkippedFrameworkSuite.pending.empty.ignored.comment PENDING comment ignored + |munit.SkippedFrameworkSuite.pending.successful.ignored started + |==> i munit.SkippedFrameworkSuite.pending.successful.ignored PENDING ignored + |munit.SkippedFrameworkSuite.pending.successful.ignored.comment started + |==> i munit.SkippedFrameworkSuite.pending.successful.ignored.comment PENDING comment ignored + |munit.SkippedFrameworkSuite.pending.failed.not-ignored started + |==> X munit.SkippedFrameworkSuite.pending.failed.not-ignored munit.FailException: tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:38 assertion failed + |37: test("pending.failed.not-ignored".pending) { + |38: assert(false) + |39: } + | at munit.FunSuite.assert(FunSuite.scala:11) + | at munit.SkippedFrameworkSuite.$anonfun$new$21(SkippedFrameworkSuite.scala:38) + |munit.SkippedFrameworkSuite.pending.failed.not-ignored.comment started + |==> X munit.SkippedFrameworkSuite.pending.failed.not-ignored.comment munit.FailException: tests/shared/src/main/scala/munit/SkippedFrameworkSuite.scala:41 assertion failed + |40: test("pending.failed.not-ignored.comment".pending("comment")) { + |41: assert(false) + |42: } + | at munit.FunSuite.assert(FunSuite.scala:11) + | at munit.SkippedFrameworkSuite.$anonfun$new$24(SkippedFrameworkSuite.scala:41) + |==> i munit.SkippedFrameworkSuite.pending.failed.ignored PENDING ignored + |==> i munit.SkippedFrameworkSuite.pending.failed.ignored.comment PENDING comment ignored + |Test run munit.SkippedFrameworkSuite finished: 2 failed, 9 ignored, 9 total, + |""".stripMargin.replace('/', File.separatorChar), format = StdoutFormat, tags = Set(OnlyJVM), arguments = Array("-v")