From 94a596c5a35b0f73d7e220d5916a166a94f37ce7 Mon Sep 17 00:00:00 2001 From: Olafur Pall Geirsson Date: Sun, 8 Mar 2020 15:45:31 +0000 Subject: [PATCH] Use consistent solution to customize tests, suites and values Previously, users could override several methods to customize how tests were transformed, how suites filtered tests and how values got lifted into futures. While this system worked OK, it wasn't easy to explain since it was quite inconsistent. Now, this commit introduces three consistent solutions to transform: * test cases: a function `Test => Test` * test suites: a function `List[Test] => List[Test]` * test values: a partial function `Any => Future[Any]` Users register new transforms using a similar API as you register fixtures: ```scala override def munitTestTransforms: List[TestTransform] = super.munitTestTransforms ++ List( new TestTransform("append scala version", { test => test.withName(test.name + "-" + scalaVersion) }) ) ``` --- docs/tests.md | 38 ++-- .../src/main/scala/munit/FunFixtures.scala | 51 +++++ .../src/main/scala/munit/FunSuite.scala | 185 ++---------------- .../src/main/scala/munit/GenericTest.scala | 4 + .../main/scala/munit/SuiteTransforms.scala | 64 ++++++ .../src/main/scala/munit/TestTransforms.scala | 75 +++++++ .../main/scala/munit/ValueTransforms.scala | 50 +++++ .../munit/ScalaVersionFrameworkSuite.scala | 10 +- .../src/test/scala/munit/BaseSuite.scala | 37 ++-- .../test/scala/munit/LazyFutureSuite.scala | 12 +- website/blog/2020-02-01-hello-world.md | 22 ++- 11 files changed, 327 insertions(+), 221 deletions(-) create mode 100644 munit/shared/src/main/scala/munit/FunFixtures.scala create mode 100644 munit/shared/src/main/scala/munit/SuiteTransforms.scala create mode 100644 munit/shared/src/main/scala/munit/TestTransforms.scala create mode 100644 munit/shared/src/main/scala/munit/ValueTransforms.scala diff --git a/docs/tests.md b/docs/tests.md index b583401f..482a301f 100644 --- a/docs/tests.md +++ b/docs/tests.md @@ -92,11 +92,11 @@ to add make sure that `LazyFuture.run()` gets called. ```scala mdoc import scala.concurrent.ExecutionContext.Implicits.global class TaskSuite extends munit.FunSuite { - override def munitTestValue(testValue: => Any): Future[Any] = - super.munitTestValue(testValue).flatMap { + override def munitValueTransforms = super.munitValueTransforms ++ List( + new ValueTransform("LazyFuture", { case LazyFuture(run) => run() - case value => Future.successful(value) - } + }) + ) implicit val ec = ExecutionContext.global test("ok-task") { LazyFuture { @@ -198,15 +198,17 @@ evaluate the body multiple times. ```scala mdoc case class Rerun(count: Int) extends munit.Tag("Rerun") class MyWindowsSuite extends munit.FunSuite { - override def munitRunTest(options: munit.TestOptions, body: () => Future[Any]): Future[Any] = { - val rerunCount = options.tags.collectFirst { - case Rerun(n) => n - }.getOrElse(1) - val futures: Seq[Future[Any]] = 1.to(rerunCount).map(_ => - super.munitRunTest(options, body)) - val result: Future[Seq[Any]] = Future.sequence(futures) - result - } + override def munitTestTransforms = super.munitTestTransforms ++ List( + new TestTransform("Rerun", { test => + val rerunCount = test.tags.collectFirst { + case Rerun(n) => n + }.getOrElse(1) + test.withBody[TestValue](() => { + val futures: Seq[Future[Any]] = 1.to(rerunCount).map(_ => test.body()) + Future.sequence(futures) + }) + }), + ) test("files".tag(Rerun(10))) { println("Hello") // will run 10 times } @@ -229,8 +231,11 @@ condition. ```scala mdoc class ScalaVersionSuite extends munit.FunSuite { val scalaVersion = scala.util.Properties.versionNumberString - override def munitNewTest(test: Test): Test = - test.withName(test.name + "-" + scalaVersion) + override def munitTestTransforms = super.munitTestTransforms ++ List( + new TestTransform("append Scala version", { test => + test.withName(test.name + "-" + scalaVersion) + }) + ) test("foo") { assert(!scalaVersion.startsWith("2.11")) } @@ -295,8 +300,7 @@ in your project. ```scala mdoc abstract class BaseSuite extends munit.FunSuite { override val munitTimeout = Duration(1, "min") - override def munitTestValue(value: => Any): Future[Any] = - ??? + override def munitTestTransforms = super.munitTestTransforms ++ List(???) // ... } class MyFirstSuite extends BaseSuite { /* ... */ } diff --git a/munit/shared/src/main/scala/munit/FunFixtures.scala b/munit/shared/src/main/scala/munit/FunFixtures.scala new file mode 100644 index 00000000..97f17c8c --- /dev/null +++ b/munit/shared/src/main/scala/munit/FunFixtures.scala @@ -0,0 +1,51 @@ +package munit + +trait FunFixtures { self: FunSuite => + + class FunFixture[T]( + val setup: TestOptions => T, + val teardown: T => Unit + ) { + def test(options: TestOptions)( + body: T => Any + )(implicit loc: Location): Unit = { + self.test(options) { + val argument = setup(options) + try body(argument) + finally teardown(argument) + }(loc) + } + } + object FunFixture { + def map2[A, B](a: FunFixture[A], b: FunFixture[B]): FunFixture[(A, B)] = + new FunFixture[(A, B)]( + setup = { options => + (a.setup(options), b.setup(options)) + }, + teardown = { + case (argumentA, argumentB) => + try a.teardown(argumentA) + finally b.teardown(argumentB) + } + ) + def map3[A, B, C]( + a: FunFixture[A], + b: FunFixture[B], + c: FunFixture[C] + ): FunFixture[(A, B, C)] = + new FunFixture[(A, B, C)]( + setup = { options => + (a.setup(options), b.setup(options), c.setup(options)) + }, + teardown = { + case (argumentA, argumentB, argumentC) => + try a.teardown(argumentA) + finally { + try b.teardown(argumentB) + finally c.teardown(argumentC) + } + } + ) + } + +} diff --git a/munit/shared/src/main/scala/munit/FunSuite.scala b/munit/shared/src/main/scala/munit/FunSuite.scala index 04f374d4..ce76f976 100644 --- a/munit/shared/src/main/scala/munit/FunSuite.scala +++ b/munit/shared/src/main/scala/munit/FunSuite.scala @@ -1,96 +1,39 @@ package munit -import munit.internal.console.StackTraces -import munit.internal.FutureCompat._ - import scala.collection.mutable -import scala.util.Failure -import scala.util.Success import scala.concurrent.Future import scala.util.control.NonFatal -import scala.util.Try -import scala.concurrent.duration.Duration -import munit.internal.PlatformCompat abstract class FunSuite extends Suite with Assertions - with TestOptionsConversions { self => + with FunFixtures + with TestOptionsConversions + with TestTransforms + with SuiteTransforms + with ValueTransforms { self => final type TestValue = Future[Any] - def isCI: Boolean = "true" == System.getenv("CI") - def munitIgnore: Boolean = false - def munitFlakyOK: Boolean = "true" == System.getenv("MUNIT_FLAKY_OK") - - private val defaultTimeout = Duration(30, "s") - def munitTimeout: Duration = defaultTimeout - val munitTestsBuffer: mutable.ArrayBuffer[Test] = - mutable.ArrayBuffer.empty[Test] + final val munitTestsBuffer: mutable.ListBuffer[Test] = + mutable.ListBuffer.empty[Test] def munitTests(): Seq[Test] = { - if (munitIgnore) { - Nil - } else { - val onlyTests = munitTestsBuffer.filter(_.tags(Only)) - if (onlyTests.nonEmpty) { - if (isCI) { - onlyTests.toSeq.map(t => - if (t.tags(Only)) { - t.withBody[TestValue](() => - fail("'Only' tag is not allowed when `isCI=true`")(t.location) - ) - } else { - t - } - ) - } else { - onlyTests.toSeq - } - } else { - munitTestsBuffer.toSeq - } - } + munitSuiteTransform(munitTestsBuffer.toList) } - def munitTestValue(testValue: => Any): Future[Any] = { - // Takes an arbitrarily nested future `Future[Future[Future[...]]]` and - // returns a `Future[T]` where `T` is not a `Future`. - def flattenFuture(future: Future[_]): Future[_] = { - val nested = future.map { - case f: Future[_] => flattenFuture(f) - case x => Future.successful(x) - }(munitExecutionContext) - nested.flattenCompat(munitExecutionContext) - } - val wrappedFuture = Future.fromTry(Try(StackTraces.dropOutside(testValue))) - val flatFuture = flattenFuture(wrappedFuture) - val awaitedFuture = PlatformCompat.waitAtMost(flatFuture, munitTimeout) - awaitedFuture - } - - def munitNewTest(test: Test): Test = - test - - def test(name: String)( - body: => Any - )(implicit loc: Location): Unit = { + def test(name: String)(body: => Any)(implicit loc: Location): Unit = { test(new TestOptions(name, Set.empty, loc))(body) } - - def test(options: TestOptions)( - body: => Any - )(implicit loc: Location): Unit = { - munitTestsBuffer += munitNewTest( + def test(options: TestOptions)(body: => Any)(implicit loc: Location): Unit = { + munitTestsBuffer += munitTestTransform( new Test( options.name, { () => - munitRunTest(options, () => { - try { - munitTestValue(body) - } catch { - case NonFatal(e) => - Future.failed(e) - } - }) + try { + munitValueTransform(body) + } catch { + case NonFatal(e) => + Future.failed(e) + } }, options.tags.toSet, loc @@ -98,98 +41,4 @@ abstract class FunSuite ) } - def munitRunTest( - options: TestOptions, - body: () => Future[Any] - ): Future[Any] = { - if (options.tags(Fail)) { - munitExpectFailure(options, body) - } else if (options.tags(Flaky)) { - munitFlaky(options, body) - } else { - body() - } - } - - def munitFlaky( - options: TestOptions, - body: () => Future[Any] - ): Future[Any] = { - body().transformCompat { - case Success(value) => Success(value) - case Failure(exception) => - if (munitFlakyOK) { - Success(new TestValues.FlakyFailure(exception)) - } else { - throw exception - } - }(munitExecutionContext) - } - - def munitExpectFailure( - options: TestOptions, - body: () => Future[Any] - ): Future[Any] = { - body().transformCompat { - case Success(value) => - Failure( - throw new FailException( - munitLines.formatLine( - options.location, - "expected failure but test passed" - ), - options.location - ) - ) - case Failure(exception) => - Success(()) - }(munitExecutionContext) - } - - class FunFixture[T]( - val setup: TestOptions => T, - val teardown: T => Unit - ) { - def test(options: TestOptions)( - body: T => Any - )(implicit loc: Location): Unit = { - self.test(options) { - val argument = setup(options) - try body(argument) - finally teardown(argument) - }(loc) - } - } - object FunFixture { - def map2[A, B](a: FunFixture[A], b: FunFixture[B]): FunFixture[(A, B)] = - new FunFixture[(A, B)]( - setup = { options => - (a.setup(options), b.setup(options)) - }, - teardown = { - case (argumentA, argumentB) => - try a.teardown(argumentA) - finally b.teardown(argumentB) - } - ) - def map3[A, B, C]( - a: FunFixture[A], - b: FunFixture[B], - c: FunFixture[C] - ): FunFixture[(A, B, C)] = - new FunFixture[(A, B, C)]( - setup = { options => - (a.setup(options), b.setup(options), c.setup(options)) - }, - teardown = { - case (argumentA, argumentB, argumentC) => - try a.teardown(argumentA) - finally { - try b.teardown(argumentB) - finally c.teardown(argumentC) - } - } - ) - } - } diff --git a/munit/shared/src/main/scala/munit/GenericTest.scala b/munit/shared/src/main/scala/munit/GenericTest.scala index 2212f4bc..6ec3b904 100644 --- a/munit/shared/src/main/scala/munit/GenericTest.scala +++ b/munit/shared/src/main/scala/munit/GenericTest.scala @@ -29,6 +29,10 @@ class GenericTest[T]( withTags(tags + newTag) def withLocation(newLocation: Location): GenericTest[T] = copy(location = newLocation) + + def withBodyMap[A](newBody: T => A): GenericTest[A] = + withBody[A](() => newBody(body())) + private[this] def copy[A]( name: String = this.name, body: () => A = this.body, diff --git a/munit/shared/src/main/scala/munit/SuiteTransforms.scala b/munit/shared/src/main/scala/munit/SuiteTransforms.scala new file mode 100644 index 00000000..21c27acc --- /dev/null +++ b/munit/shared/src/main/scala/munit/SuiteTransforms.scala @@ -0,0 +1,64 @@ +package munit + +import scala.concurrent.Future +import scala.util.control.NonFatal + +trait SuiteTransforms { this: FunSuite => + + final class SuiteTransform(val name: String, fn: List[Test] => List[Test]) + extends Function1[List[Test], List[Test]] { + def apply(v1: List[Test]): List[Test] = fn(v1) + } + + def munitSuiteTransforms: List[SuiteTransform] = + List( + munitIgnoreSuiteTransform, + munitOnlySuiteTransform + ) + + final def munitSuiteTransform(tests: List[Test]): List[Test] = { + try { + munitSuiteTransforms.foldLeft(tests) { + case (ts, fn) => fn(ts) + } + } catch { + case NonFatal(e) => + List( + new Test( + "munitSuiteTransform", + () => Future.failed(e) + )(Location.empty) + ) + } + } + + def munitIgnore: Boolean = false + final def munitIgnoreSuiteTransform: SuiteTransform = + new SuiteTransform("munitIgnore", { tests => + if (munitIgnore) Nil + else tests + }) + + def isCI: Boolean = "true" == System.getenv("CI") + final def munitOnlySuiteTransform: SuiteTransform = + new SuiteTransform("only", { tests => + val onlySuite = tests.filter(_.tags(Only)) + if (onlySuite.nonEmpty) { + if (!isCI) { + onlySuite + } else { + onlySuite.map(t => + if (t.tags(Only)) { + t.withBody[TestValue](() => + fail("'Only' tag is not allowed when `isCI=true`")(t.location) + ) + } else { + t + } + ) + } + } else { + tests + } + }) +} diff --git a/munit/shared/src/main/scala/munit/TestTransforms.scala b/munit/shared/src/main/scala/munit/TestTransforms.scala new file mode 100644 index 00000000..af8e4ecd --- /dev/null +++ b/munit/shared/src/main/scala/munit/TestTransforms.scala @@ -0,0 +1,75 @@ +package munit + +import munit.internal.FutureCompat._ +import scala.util.Success +import scala.util.Failure +import scala.concurrent.Future +import scala.util.control.NonFatal + +trait TestTransforms { this: FunSuite => + + final class TestTransform(val name: String, fn: Test => Test) + extends Function1[Test, Test] { + def apply(v1: Test): Test = fn(v1) + } + + def munitTestTransforms: List[TestTransform] = + List( + munitFailTransform, + munitFlakyTransform + ) + + final def munitTestTransform(test: Test): Test = { + try { + munitTestTransforms.foldLeft(test) { + case (t, fn) => fn(t) + } + } catch { + case NonFatal(e) => + test.withBody[TestValue](() => Future.failed(e)) + } + } + + final def munitFailTransform: TestTransform = + new TestTransform("fail", { t => + if (t.tags(Fail)) { + t.withBodyMap[TestValue]( + _.transformCompat { + case Success(value) => + Failure( + throw new FailException( + munitLines.formatLine( + t.location, + "expected failure but test passed" + ), + t.location + ) + ) + case Failure(exception) => + Success(()) + }(munitExecutionContext) + ) + } else { + t + } + }) + + def munitFlakyOK: Boolean = "true" == System.getenv("MUNIT_FLAKY_OK") + final def munitFlakyTransform: TestTransform = + new TestTransform("flaky", { t => + if (t.tags(Flaky)) { + t.withBodyMap(_.transformCompat { + case Success(value) => Success(value) + case Failure(exception) => + if (munitFlakyOK) { + Success(new TestValues.FlakyFailure(exception)) + } else { + throw exception + } + }(munitExecutionContext)) + } else { + t + } + }) + +} diff --git a/munit/shared/src/main/scala/munit/ValueTransforms.scala b/munit/shared/src/main/scala/munit/ValueTransforms.scala new file mode 100644 index 00000000..4c2ff54a --- /dev/null +++ b/munit/shared/src/main/scala/munit/ValueTransforms.scala @@ -0,0 +1,50 @@ +package munit + +import scala.concurrent.Future +import munit.internal.FutureCompat._ +import scala.util.Try +import munit.internal.console.StackTraces +import munit.internal.PlatformCompat +import scala.concurrent.duration.Duration +import scala.concurrent.duration.FiniteDuration +import java.util.concurrent.TimeUnit + +trait ValueTransforms { this: FunSuite => + + final class ValueTransform( + val name: String, + fn: PartialFunction[Any, Future[Any]] + ) extends Function1[Any, Option[Future[Any]]] { + def apply(v1: Any): Option[Future[Any]] = fn.lift(v1) + } + + def munitValueTransforms: List[ValueTransform] = + List( + munitFutureTransform + ) + + def munitTimeout: Duration = new FiniteDuration(30, TimeUnit.SECONDS) + final def munitValueTransform(testValue: => Any): Future[Any] = { + // Takes an arbitrarily nested future `Future[Future[Future[...]]]` and + // returns a `Future[T]` where `T` is not a `Future`. + def flattenFuture(future: Future[_]): Future[_] = { + val nested: Future[Future[Any]] = future.map { value => + val transformed = munitValueTransforms.iterator + .map(fn => fn(value)) + .collectFirst { case Some(future) => future } + transformed match { + case Some(f) => flattenFuture(f) + case None => Future.successful(value) + } + }(munitExecutionContext) + nested.flattenCompat(munitExecutionContext) + } + val wrappedFuture = Future.fromTry(Try(StackTraces.dropOutside(testValue))) + val flatFuture = flattenFuture(wrappedFuture) + val awaitedFuture = PlatformCompat.waitAtMost(flatFuture, munitTimeout) + awaitedFuture + } + + final def munitFutureTransform: ValueTransform = + new ValueTransform("Future", { case e: Future[_] => e }) +} diff --git a/tests/shared/src/main/scala/munit/ScalaVersionFrameworkSuite.scala b/tests/shared/src/main/scala/munit/ScalaVersionFrameworkSuite.scala index a62f012d..1d18c8ee 100644 --- a/tests/shared/src/main/scala/munit/ScalaVersionFrameworkSuite.scala +++ b/tests/shared/src/main/scala/munit/ScalaVersionFrameworkSuite.scala @@ -2,8 +2,14 @@ package munit class ScalaVersionFrameworkSuite extends munit.FunSuite { val scalaVersion = "2.12.100" - override def munitNewTest(test: Test): Test = - test.withName(test.name + "-" + scalaVersion) + + override def munitTestTransforms: List[TestTransform] = + super.munitTestTransforms ++ List( + new TestTransform("append scala version", { test => + test.withName(test.name + "-" + scalaVersion) + }) + ) + test("foo") { assertEquals(List(1).head, 1) } diff --git a/tests/shared/src/test/scala/munit/BaseSuite.scala b/tests/shared/src/test/scala/munit/BaseSuite.scala index 4dc90316..a16030f3 100644 --- a/tests/shared/src/test/scala/munit/BaseSuite.scala +++ b/tests/shared/src/test/scala/munit/BaseSuite.scala @@ -1,25 +1,24 @@ package munit import munit.internal.PlatformCompat -import scala.concurrent.Future class BaseSuite extends FunSuite { - override def munitRunTest( - options: TestOptions, - body: () => Future[Any] - ): Future[Any] = { - def isDotty: Boolean = - BuildInfo.scalaVersion.startsWith("0.") - def is213: Boolean = - BuildInfo.scalaVersion.startsWith("2.13") || isDotty - if (options.tags(NoDotty) && isDotty) { - Future.successful(Ignore) - } else if (options.tags(Only213) && !is213) { - Future.successful(Ignore) - } else if (options.tags(OnlyJVM) && !PlatformCompat.isJVM) { - Future.successful(Ignore) - } else { - super.munitRunTest(options, body) - } - } + override def munitTestTransforms: List[TestTransform] = + super.munitTestTransforms ++ List( + new TestTransform("BaseSuite", { test => + def isDotty: Boolean = + BuildInfo.scalaVersion.startsWith("0.") + def is213: Boolean = + BuildInfo.scalaVersion.startsWith("2.13") || isDotty + if (test.tags(NoDotty) && isDotty) { + test.tag(Ignore) + } else if (test.tags(Only213) && !is213) { + test.tag(Ignore) + } else if (test.tags(OnlyJVM) && !PlatformCompat.isJVM) { + test.tag(Ignore) + } else { + test + } + }) + ) } diff --git a/tests/shared/src/test/scala/munit/LazyFutureSuite.scala b/tests/shared/src/test/scala/munit/LazyFutureSuite.scala index b695af96..42b0810a 100644 --- a/tests/shared/src/test/scala/munit/LazyFutureSuite.scala +++ b/tests/shared/src/test/scala/munit/LazyFutureSuite.scala @@ -10,11 +10,13 @@ class LazyFutureSuite extends FunSuite { def apply[T](thunk: => T)(implicit ec: ExecutionContext): LazyFuture[T] = LazyFuture(() => Future(thunk)) } - override def munitTestValue(testValue: => Any): Future[Any] = - super.munitTestValue(testValue).flatMap { - case LazyFuture(run) => run() - case value => Future.successful(value) - } + + override def munitValueTransforms: List[ValueTransform] = + super.munitValueTransforms ++ List( + new ValueTransform("LazyFuture", { + case LazyFuture(run) => run() + }) + ) test("ok-task".fail) { LazyFuture { diff --git a/website/blog/2020-02-01-hello-world.md b/website/blog/2020-02-01-hello-world.md index fd72201c..fe31ccd6 100644 --- a/website/blog/2020-02-01-hello-world.md +++ b/website/blog/2020-02-01-hello-world.md @@ -127,16 +127,17 @@ import scala.util.Properties import munit._ object Windows213 extends Tag("Windows213") class MySuite extends FunSuite { - // reminder: type Test = GenericTest[Future[Any]] - override def munitNewTest(test: Test): Test = { - val isIgnored = - test.tags(Windows213) && !( - Properties.isWin && - Properties.versionNumberString.startsWith("2.13") - ) - if (isIgnored) test.tag(Ignore) - else test - } + override def munitTestTransforms = super.munitTestTransforms ++ List( + new TestTransform("Windows213", { test => + val isIgnored = + test.tags(Windows213) && !( + Properties.isWin && + Properties.versionNumberString.startsWith("2.13") + ) + if (isIgnored) test.tag(Ignore) + else test + }) + ) test("windows-213".tag(Windows213)) { // Only runs when operating system is Windows and Scala version is 2.13 @@ -145,6 +146,7 @@ class MySuite extends FunSuite { // Always runs like a normal test. } } + ``` By encoding the environment requirements in the test implementation, we prevent