From fd552a8f8a8fd699793f77f222bfff9de77307fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 30 Jun 2021 23:17:49 +0200 Subject: [PATCH 1/4] Expose test filters as standalone tool --- settings.gradle.kts | 1 + tool/filter/README.md | 9 + tool/filter/build.gradle.kts | 19 + .../src/main/kotlin/flank/filter/Filter.kt | 13 + .../main/kotlin/flank/filter/internal/Data.kt | 52 +++ .../filter/internal/FilterFromTestTarget.kt | 52 +++ .../flank/filter/internal/factory/AllOf.kt | 12 + .../flank/filter/internal/factory/AnyOf.kt | 10 + .../filter/internal/factory/FromTestFile.kt | 20 + .../flank/filter/internal/factory/Not.kt | 11 + .../filter/internal/factory/WithAnnotation.kt | 11 + .../filter/internal/factory/WithClassName.kt | 27 ++ .../internal/factory/WithPackageName.kt | 12 + .../flank/filter/internal/factory/WithSize.kt | 28 ++ .../test/kotlin/flank/filter/FilterKtTest.kt | 420 ++++++++++++++++++ .../src/test/resources/dummy-tests-file.txt | 2 + .../src/test/resources/exclude-tests.txt | 2 + 17 files changed, 701 insertions(+) create mode 100644 tool/filter/README.md create mode 100644 tool/filter/build.gradle.kts create mode 100644 tool/filter/src/main/kotlin/flank/filter/Filter.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/Data.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/FilterFromTestTarget.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/factory/AllOf.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/factory/AnyOf.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/factory/FromTestFile.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/factory/Not.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/factory/WithAnnotation.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/factory/WithClassName.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/factory/WithPackageName.kt create mode 100644 tool/filter/src/main/kotlin/flank/filter/internal/factory/WithSize.kt create mode 100644 tool/filter/src/test/kotlin/flank/filter/FilterKtTest.kt create mode 100644 tool/filter/src/test/resources/dummy-tests-file.txt create mode 100644 tool/filter/src/test/resources/exclude-tests.txt diff --git a/settings.gradle.kts b/settings.gradle.kts index 2e616e754d..6bac4f6874 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -22,6 +22,7 @@ include( ":corellium:sandbox", ":tool:apk", + ":tool:filter", ":tool:shard", ":tool:shard:calculate", ":tool:shard:obfuscate", diff --git a/tool/filter/README.md b/tool/filter/README.md new file mode 100644 index 0000000000..f9ce5c9331 --- /dev/null +++ b/tool/filter/README.md @@ -0,0 +1,9 @@ +# Filtering test targets + +Tool for filtering test cases using predefined test targets. + +### References + +* Module type - [tool](../../../docs/architecture.md#tool) +* Dependency type - [static](../../../docs/architecture.md#static_dependencies) +* Public API - [Filter.kt](./src/main/kotlin/flank/filter/Filter.kt) diff --git a/tool/filter/build.gradle.kts b/tool/filter/build.gradle.kts new file mode 100644 index 0000000000..1417115155 --- /dev/null +++ b/tool/filter/build.gradle.kts @@ -0,0 +1,19 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin(Plugins.Kotlin.PLUGIN_JVM) +} + +repositories { + jcenter() + mavenCentral() + maven(url = "https://kotlin.bintray.com/kotlinx") +} + +tasks.withType { kotlinOptions.jvmTarget = "1.8" } + +dependencies { + implementation(Dependencies.KOTLIN_COROUTINES_CORE) + + testImplementation(Dependencies.JUNIT) +} diff --git a/tool/filter/src/main/kotlin/flank/filter/Filter.kt b/tool/filter/src/main/kotlin/flank/filter/Filter.kt new file mode 100644 index 0000000000..a9356e28de --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/Filter.kt @@ -0,0 +1,13 @@ +package flank.filter + +import flank.filter.internal.fromTestTargets + +/** + * Factory for creating [ShouldRun] function from given [targets]. + */ +fun createTestCasesFilter(targets: List): ShouldRun = + fromTestTargets(targets).shouldRun + +typealias ShouldRun = (Target) -> Boolean + +typealias Target = Pair> diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/Data.kt b/tool/filter/src/main/kotlin/flank/filter/internal/Data.kt new file mode 100644 index 0000000000..45cfd12ef7 --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/Data.kt @@ -0,0 +1,52 @@ +package flank.filter.internal + +import flank.filter.ShouldRun + +internal object Test { + + /** + * TestFilter similar to https://junit.org/junit4/javadoc/4.12/org/junit/runner/manipulation/Filter.html + * + * Annotations are tracked as all annotation filters must match on a test. + * + * @property describe - description of the filter + * @property shouldRun - lambda that returns if a TestMethod should be included in the test run + * **/ + data class Filter( + val describe: String, + val shouldRun: ShouldRun, + val isAnnotation: Boolean = false + ) + + /** + * Supports arguments defined in androidx.test.internal.runner.RunnerArgs + * + * Multiple annotation arguments will result in the intersection. + * https://developer.android.com/reference/android/support/test/runner/AndroidJUnitRunner + * https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run + */ + internal object Target { + + object Type { + const val ARGUMENT_TEST_CLASS = "class" + const val ARGUMENT_NOT_TEST_CLASS = "notClass" + + const val ARGUMENT_TEST_SIZE = "size" + + const val ARGUMENT_ANNOTATION = "annotation" + const val ARGUMENT_NOT_ANNOTATION = "notAnnotation" + + const val ARGUMENT_TEST_PACKAGE = "package" + const val ARGUMENT_NOT_TEST_PACKAGE = "notPackage" + + const val ARGUMENT_TEST_FILE = "testFile" + const val ARGUMENT_NOT_TEST_FILE = "notTestFile" + } + + object Size { + const val LARGE = "large" + const val MEDIUM = "medium" + const val SMALL = "small" + } + } +} diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/FilterFromTestTarget.kt b/tool/filter/src/main/kotlin/flank/filter/internal/FilterFromTestTarget.kt new file mode 100644 index 0000000000..3ac24988a8 --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/FilterFromTestTarget.kt @@ -0,0 +1,52 @@ +package flank.filter.internal + +import flank.filter.Target +import flank.filter.internal.factory.allOf +import flank.filter.internal.factory.anyOf +import flank.filter.internal.factory.fromTestFile +import flank.filter.internal.factory.not +import flank.filter.internal.factory.withAnnotation +import flank.filter.internal.factory.withClassName +import flank.filter.internal.factory.withPackageName +import flank.filter.internal.factory.withSize + +internal fun fromTestTargets(testTargets: List): Test.Filter { + val parsedFilters: List = testTargets + .asSequence() + .map(String::trim) + .map { parseTarget(it).createFilter() } + .toList() + + // select test method name filters and short circuit if they match ex: class a.b#c + val annotationFilters = parsedFilters.filter { it.isAnnotation }.toTypedArray() + val otherFilters = parsedFilters.filterNot { it.isAnnotation } + val exclude = otherFilters.filter { it.describe.startsWith("not") }.toTypedArray() + val include = otherFilters.filterNot { it.describe.startsWith("not") }.toTypedArray() + + return allOf(*annotationFilters, *exclude, anyOf(*include)) +} + +private fun parseTarget(target: String): Target = + target.split(" ", limit = 2).run { + require(size == 2) { "Invalid argument: $target" } + first() to last().split(",").map(String::trim).apply { + require(isNotEmpty()) { "Empty args parsed from $target" } + } + } + +private fun Target.createFilter(): Test.Filter = + when (type) { + Test.Target.Type.ARGUMENT_TEST_CLASS -> withClassName(args) + Test.Target.Type.ARGUMENT_NOT_TEST_CLASS -> not(withClassName(args)) + Test.Target.Type.ARGUMENT_TEST_PACKAGE -> withPackageName(args) + Test.Target.Type.ARGUMENT_NOT_TEST_PACKAGE -> not(withPackageName(args)) + Test.Target.Type.ARGUMENT_ANNOTATION -> withAnnotation(args) + Test.Target.Type.ARGUMENT_NOT_ANNOTATION -> not(withAnnotation(args)) + Test.Target.Type.ARGUMENT_TEST_FILE -> fromTestFile(args) + Test.Target.Type.ARGUMENT_NOT_TEST_FILE -> not(fromTestFile(args)) + Test.Target.Type.ARGUMENT_TEST_SIZE -> withSize(args) + else -> throw IllegalArgumentException("Filtering option $type not supported") + } + +private val Target.type get() = first +private val Target.args get() = second diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/factory/AllOf.kt b/tool/filter/src/main/kotlin/flank/filter/internal/factory/AllOf.kt new file mode 100644 index 0000000000..db964b5872 --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/factory/AllOf.kt @@ -0,0 +1,12 @@ +package flank.filter.internal.factory + +import flank.filter.internal.Test + +internal fun allOf(vararg filters: Test.Filter) = Test.Filter( + describe = "allOf ${filters.map { it.describe }}", + shouldRun = { testMethod -> + filters.isEmpty() || filters.all { filter -> + filter.shouldRun(testMethod) + } + } +) diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/factory/AnyOf.kt b/tool/filter/src/main/kotlin/flank/filter/internal/factory/AnyOf.kt new file mode 100644 index 0000000000..e1ea2bb1c1 --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/factory/AnyOf.kt @@ -0,0 +1,10 @@ +package flank.filter.internal.factory + +import flank.filter.internal.Test + +internal fun anyOf(vararg filters: Test.Filter) = Test.Filter( + describe = "anyOf ${filters.map { it.describe }}", + shouldRun = { testMethod -> + filters.isEmpty() || filters.any { filter -> filter.shouldRun(testMethod) } + } +) diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/factory/FromTestFile.kt b/tool/filter/src/main/kotlin/flank/filter/internal/factory/FromTestFile.kt new file mode 100644 index 0000000000..d32050ca1e --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/factory/FromTestFile.kt @@ -0,0 +1,20 @@ +package flank.filter.internal.factory + +import flank.filter.internal.Test +import java.io.IOException +import java.nio.file.Files +import java.nio.file.Paths + +internal fun fromTestFile(args: List): Test.Filter { + require(args.size == 1) { "Invalid file path" } + val path = Paths.get(args[0]) + try { + val lines = Files.readAllLines(path) + // this is really an implementation detail: + // being the package name most generic one, it is able to filter properly if you pass the package name, + // the fully qualified class name or the fully qualified method name. + return withPackageName(lines) + } catch (e: IOException) { + throw IllegalArgumentException("Unable to read testFile", e) + } +} diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/factory/Not.kt b/tool/filter/src/main/kotlin/flank/filter/internal/factory/Not.kt new file mode 100644 index 0000000000..cc716ad9d1 --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/factory/Not.kt @@ -0,0 +1,11 @@ +package flank.filter.internal.factory + +import flank.filter.internal.Test + +internal fun not(filter: Test.Filter) = Test.Filter( + describe = "not (${filter.describe})", + shouldRun = { testMethod -> + filter.shouldRun(testMethod).not() + }, + isAnnotation = filter.isAnnotation +) diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithAnnotation.kt b/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithAnnotation.kt new file mode 100644 index 0000000000..b5546a073a --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithAnnotation.kt @@ -0,0 +1,11 @@ +package flank.filter.internal.factory + +import flank.filter.internal.Test + +internal fun withAnnotation(filter: List) = Test.Filter( + describe = "withAnnotation (${filter.joinToString(", ")})", + shouldRun = { (_, annotations) -> + annotations.any { it in filter } + }, + isAnnotation = true +) diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithClassName.kt b/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithClassName.kt new file mode 100644 index 0000000000..6b9d583dd5 --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithClassName.kt @@ -0,0 +1,27 @@ +package flank.filter.internal.factory + +import flank.filter.internal.Test + +internal fun withClassName(classNames: List): Test.Filter { + // splits foo.bar.TestClass1#testMethod1 into [foo.bar.TestClass1, testMethod1] + val classFilters = classNames.map { it.extractClassAndTestNames() } + + return Test.Filter( + describe = "withClassName (${classNames.joinToString(", ")})", + shouldRun = { (name, _) -> + name.extractClassAndTestNames().matchFilters(classFilters) + } + ) +} + +private fun String.extractClassAndTestNames() = split("#") + +private fun List.matchFilters(classFilters: List>): Boolean { + fun List.className() = first() + fun List.methodName() = last() + return classFilters.any { filter -> + // When filter.size == 1 all test methods from the class should run therefore we do not compare method names + // When filter.size != 1 only particular test from the class should be launched and we need to compare method names as well + className() == filter.className() && (filter.size == 1 || methodName() == filter.methodName()) + } +} diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithPackageName.kt b/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithPackageName.kt new file mode 100644 index 0000000000..f2905a8b23 --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithPackageName.kt @@ -0,0 +1,12 @@ +package flank.filter.internal.factory + +import flank.filter.internal.Test + +internal fun withPackageName(packageNames: List) = Test.Filter( + describe = "withPackageName (${packageNames.joinToString(", ")})", + shouldRun = { (name, _) -> + packageNames.any { packageName -> + name.startsWith(packageName) + } + } +) diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithSize.kt b/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithSize.kt new file mode 100644 index 0000000000..c91ecb159f --- /dev/null +++ b/tool/filter/src/main/kotlin/flank/filter/internal/factory/WithSize.kt @@ -0,0 +1,28 @@ +package flank.filter.internal.factory + +import flank.filter.internal.Test + +internal fun withSize(args: List): Test.Filter { + val filter = args.map(sizeAnnotations::getValue) + return Test.Filter( + describe = "withSize (${filter.joinToString(", ")})", + shouldRun = { (_, annotations) -> + // Ensure that all annotation with a name matching this size are detected, like + // https://developer.android.com/reference/android/support/test/filters/LargeTest or + // https://developer.android.com/reference/androidx/test/filters/LargeTest + annotations.any { it.split(".").last() in filter } + }, + isAnnotation = true + ) +} + +private val sizeAnnotations: Map = + setOf( + Test.Target.Size.LARGE, + Test.Target.Size.MEDIUM, + Test.Target.Size.SMALL, + ).associateWith { size -> + size.replaceFirstChar { char -> char.uppercaseChar() } + "Test" + }.withDefault { size -> + throw IllegalArgumentException("Unknown size $size") + } diff --git a/tool/filter/src/test/kotlin/flank/filter/FilterKtTest.kt b/tool/filter/src/test/kotlin/flank/filter/FilterKtTest.kt new file mode 100644 index 0000000000..f5d5296ab9 --- /dev/null +++ b/tool/filter/src/test/kotlin/flank/filter/FilterKtTest.kt @@ -0,0 +1,420 @@ +package flank.filter + +import flank.filter.internal.fromTestTargets + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.nio.file.Path +import java.nio.file.Paths + +val FOO_PACKAGE: Target = "foo.ClassName#testName" to emptyList() +val BAR_PACKAGE: Target = "bar.ClassName#testName" to emptyList() +val FOO_CLASSNAME: Target = "whatever.Foo#testName" to emptyList() +val BAR_CLASSNAME: Target = "whatever.Bar#testName" to emptyList() +val WITHOUT_IGNORE_ANNOTATION: Target = "whatever.Foo#testName" to emptyList() +val WITH_IGNORE_ANNOTATION: Target = "whatever.Foo#testName" to listOf("org.junit.Ignore") +val WITH_FOO_ANNOTATION: Target = "whatever.Foo#testName" to listOf("Foo") +val WITH_BAR_ANNOTATION: Target = "whatever.Foo#testName" to listOf("Bar") +val WITHOUT_FOO_ANNOTATION: Target = "whatever.Foo#testName" to emptyList() +val WITH_FOO_ANNOTATION_AND_PACKAGE: Target = "foo.Bar#testName" to listOf("Foo") +val WITH_LARGE_ANNOTATION: Target = "whatever.Foo#testName" to listOf("android.test.suitebuilder.annotation.LargeTest") +val WITH_MEDIUM_ANNOTATION: Target = "whatever.Foo#testName" to listOf("android.support.test.filters.MediumTest") +val WITH_SMALL_ANNOTATION: Target = "whatever.Foo#testName" to listOf("androidx.test.filters.SmallTest") +val WITHOUT_LARGE_ANNOTATION: Target = "whatever.Foo#testName" to emptyList() +val WITHOUT_MEDIUM_ANNOTATION: Target = "whatever.Foo#testName" to emptyList() +val WITHOUT_SMALL_ANNOTATION: Target = "whatever.Foo#testName" to emptyList() +const val TEST_FILE = "src/test/resources/dummy-tests-file.txt" +const val TEST_FILE_2 = "src/test/resources/exclude-tests.txt" +private const val IGNORE_ANNOTATION = "org.junit.Ignore" + +@Suppress("TooManyFunctions") +class FilterKtTest { + + private val targets = listOf( + TargetsHelper( + pack = "anyPackage_1", + cl = "anyClass_1", + m = "anyMethod_1", + annotation = IGNORE_ANNOTATION + ), + TargetsHelper(pack = "anyPackage_2", cl = "anyClass_2", m = "anyMethod_2", annotation = "Foo"), + TargetsHelper(pack = "anyPackage_3", cl = "anyClass_3", m = "anyMethod_3", annotation = "Bar"), + TargetsHelper( + pack = "anyPackage_4", + cl = "anyClass_4", + m = "anyMethod_4", + annotation = IGNORE_ANNOTATION + ) + ) + + private val testMethodSet = targets.map { getDefaultTestMethod(it.fullView, it.annotation) } + private val commonExpected = targets.map { it.fullView } + + @Test + fun testIgnoreMultipleAnnotations() { + val target: Target = "com.example.app.ExampleUiTest#testFails" to listOf( + "org.junit.runner.RunWith", + "org.junit.Ignore", + "org.junit.Test", + ) + + val filter = createTestCasesFilter(listOf("notAnnotation org.junit.Ignore")) + + assertFalse(filter(target)) + } + + @Test + fun testFilteringByPackage() { + val filter = createTestCasesFilter(listOf("package foo")) + + assertTrue(filter(FOO_PACKAGE)) + assertFalse(filter(BAR_PACKAGE)) + } + + @Test + fun testFilteringByPackageNegative() { + val filter = createTestCasesFilter(listOf("notPackage foo")) + + assertFalse(filter(FOO_PACKAGE)) + assertTrue(filter(BAR_PACKAGE)) + } + + @Test + fun testFilteringByClassName() { + val filter = createTestCasesFilter(listOf("class whatever.Foo")) + + assertTrue(filter(FOO_CLASSNAME)) + assertFalse(filter(BAR_CLASSNAME)) + } + + @Test + fun testFilteringByClassNameNegative() { + val filter = createTestCasesFilter(listOf("notClass whatever.Foo")) + + assertFalse(filter(FOO_CLASSNAME)) + assertTrue(filter(BAR_CLASSNAME)) + } + + @Test + fun `empty targets should not filter @Ignore annotated tests`() { + val filter = createTestCasesFilter(listOf()) + + assertTrue(filter(WITH_IGNORE_ANNOTATION)) + assertTrue(filter(WITHOUT_IGNORE_ANNOTATION)) + } + + @Test + fun testFilteringByAnnotation() { + val filter = createTestCasesFilter(listOf("annotation Foo,Bar")) + + assertTrue(filter(WITH_FOO_ANNOTATION)) + assertTrue(filter(WITH_BAR_ANNOTATION)) + assertFalse(filter(WITHOUT_FOO_ANNOTATION)) + } + + @Test + fun testFilteringByAnnotationWithSpaces() { + val filter = createTestCasesFilter(listOf("annotation Foo, Bar")) + + assertTrue(filter(WITH_FOO_ANNOTATION)) + assertTrue(filter(WITH_BAR_ANNOTATION)) + assertFalse(filter(WITHOUT_FOO_ANNOTATION)) + } + + @Test + fun testFilteringBySizeLarge() { + val filter = createTestCasesFilter(listOf("size large")) + + assertTrue(filter(WITH_LARGE_ANNOTATION)) + assertFalse(filter(WITHOUT_LARGE_ANNOTATION)) + } + + @Test + fun testFilteringBySizeMedium() { + val filter = createTestCasesFilter(listOf("size medium")) + + assertTrue(filter(WITH_MEDIUM_ANNOTATION)) + assertFalse(filter(WITHOUT_MEDIUM_ANNOTATION)) + } + + @Test + fun testFilteringBySizeSmall() { + val filter = createTestCasesFilter(listOf("size small")) + + assertTrue(filter(WITH_SMALL_ANNOTATION)) + assertFalse(filter(WITHOUT_SMALL_ANNOTATION)) + } + + @Test(expected = IllegalArgumentException::class) + fun testFilteringBySizeInvalidWillThrowException() { + createTestCasesFilter(listOf("size foo")) + } + + @Test + fun testFilteringBySizes() { + val filter = createTestCasesFilter(listOf("size large,small")) + + assertTrue(filter(WITH_LARGE_ANNOTATION)) + assertTrue(filter(WITH_SMALL_ANNOTATION)) + assertFalse(filter(WITHOUT_LARGE_ANNOTATION)) + assertFalse(filter(WITHOUT_SMALL_ANNOTATION)) + } + + @Test + fun testFilteringBySizesWithSpace() { + val filter = createTestCasesFilter(listOf("size large, small")) + + assertTrue(filter(WITH_LARGE_ANNOTATION)) + assertTrue(filter(WITH_SMALL_ANNOTATION)) + assertFalse(filter(WITHOUT_LARGE_ANNOTATION)) + assertFalse(filter(WITHOUT_SMALL_ANNOTATION)) + } + + @Test(expected = IllegalArgumentException::class) + fun testFilteringByNotSizesWillThrowException() { + createTestCasesFilter(listOf("notSize large")) + } + + @Test + fun testFilteringByAnnotationNegative() { + val filter = createTestCasesFilter(listOf("notAnnotation Foo")) + + assertFalse(filter(WITH_FOO_ANNOTATION)) + assertTrue(filter(WITHOUT_FOO_ANNOTATION)) + } + + @Test + fun allOfProperlyChecksAllFilters() { + val filter = createTestCasesFilter(listOf("package foo,bar", "annotation Foo")) + + assertFalse(filter(FOO_PACKAGE)) + assertFalse(filter(BAR_PACKAGE)) + assertFalse(filter(WITH_FOO_ANNOTATION)) + assertTrue(filter(WITH_FOO_ANNOTATION_AND_PACKAGE)) + } + + @Test + fun testFilteringFromFileNegative() { + val file = getPath(TEST_FILE) + val filePath = file.toString() + + val filter = createTestCasesFilter(listOf("testFile $filePath")) + + assertTrue(filter(FOO_PACKAGE)) + assertTrue(filter(BAR_PACKAGE)) + } + + @Test + fun testFilteringFromFile() { + val file = getPath(TEST_FILE) + val filePath = file.toString() + + val filter = createTestCasesFilter(listOf("notTestFile $filePath")) + + assertFalse(filter(FOO_PACKAGE)) + assertFalse(filter(BAR_PACKAGE)) + } + + @Test(expected = IllegalArgumentException::class) + fun passingMalformedCommandWillThrowException() { + createTestCasesFilter(listOf("class=com.my.package")) + } + + @Test(expected = IllegalArgumentException::class) + fun passingInvalidCommandWillThrowException() { + createTestCasesFilter(listOf("invalidCommand com.my.package")) + } + + @Test + fun classFilterOverridesNotAnnotation() { + val filter = fromTestTargets( + listOf( + "notAnnotation Foo", + "class anyPackage_2.anyClass_2#anyMethod_2", + "class anyPackage_3.anyClass_3#anyMethod_3" + ) + ) + + val output = mutableListOf() + val filtered = testMethodSet.asSequence().filter { test -> + val result = filter.shouldRun(test) + output.add("""$result ${test.first} [${filter.describe}]""") + result + }.map { "class ${it.first}" }.toList() + + val expected = listOf( + "false anyPackage_1.anyClass_1#anyMethod_1 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]", + "false anyPackage_2.anyClass_2#anyMethod_2 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]", + "true anyPackage_3.anyClass_3#anyMethod_3 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]", + "false anyPackage_4.anyClass_4#anyMethod_4 [allOf [not (withAnnotation (Foo)), anyOf [withClassName (anyPackage_2.anyClass_2#anyMethod_2), withClassName (anyPackage_3.anyClass_3#anyMethod_3)]]]" + ) + + assertEquals(expected, output) + assertEquals(listOf("class anyPackage_3.anyClass_3#anyMethod_3"), filtered) + } + + @Test + fun notAnnotationFiltersWithClass() { + val filter = createTestCasesFilter(listOf("notAnnotation Foo", "class anyPackage_1.anyClass_1#anyMethod_1")) + val filtered = testMethodSet.withFilter(filter) + + assertEquals(listOf("anyPackage_1.anyClass_1#anyMethod_1"), filtered) + } + + @Test + fun notAnnotationFilters() { + val filter = createTestCasesFilter(listOf("notAnnotation Moo")) + val filtered = testMethodSet.withFilter(filter, enrich = false) + + assertEquals(commonExpected, filtered) + } + + @Test + fun methodOverrideIgnored() { + val filter = createTestCasesFilter(targets.map { it.methodView }) + val filtered = testMethodSet.withFilter(filter) + + assertEquals(commonExpected, filtered) + } + + @Test + fun multipleClassesResolveToMethods() { + val filter = createTestCasesFilter(targets.map { it.classView }) + val filtered = testMethodSet.withFilter(filter) + + assertEquals(commonExpected, filtered) + } + + @Test + fun multiplePackagesResolveToMethods() { + val filter = createTestCasesFilter(targets.map { it.packageView }) + val filtered = testMethodSet.withFilter(filter) + + assertEquals(commonExpected, filtered) + } + + @Test + fun `by default - should contain method annotated with @Ignore`() { + val byPackageFilter = createTestCasesFilter(targets.map { it.packageView }) + val byClassFilter = createTestCasesFilter(targets.map { it.classView }) + val byNotAnnotationFilter = createTestCasesFilter(listOf("notAnnotation ThereIsNoSuchAnnotation")) + + val byPackage = testMethodSet.withFilter(byPackageFilter) + val byClass = testMethodSet.withFilter(byClassFilter) + val byNotAnnotation = testMethodSet.withFilter(byNotAnnotationFilter, enrich = false) + + assertEquals(commonExpected, byPackage) + assertEquals(commonExpected, byClass) + assertEquals(commonExpected, byNotAnnotation) + } + + @Test + fun `should filter tests annotated with @Ignore if user explicitly want to do so`() { + val byNotAnnotationFilter = createTestCasesFilter(listOf("notAnnotation $IGNORE_ANNOTATION")) + val byNotAnnotation = testMethodSet.withFilter(byNotAnnotationFilter, enrich = false) + val expected = targets.filterNot { it.annotation == IGNORE_ANNOTATION }.map { it.fullView } + + assertEquals(byNotAnnotation, expected) + } + + @Test + fun testFilteringClassAndPackageNegative() { + val filter = createTestCasesFilter(listOf("notPackage foo", "notClass whatever.Bar")) + + assertFalse(filter(FOO_PACKAGE)) + assertFalse(filter(BAR_CLASSNAME)) + assertTrue(filter(BAR_PACKAGE)) + } + + @Test + fun testFilteringClassAndPackageNegativeFromFile() { + val file = getPath(TEST_FILE_2) // contents: foo whatever.Bar + val filePath = file.toString() + + val filter = createTestCasesFilter(listOf("notTestFile $filePath")) + + assertFalse(filter(FOO_PACKAGE)) + assertFalse(filter(BAR_CLASSNAME)) + assertTrue(filter(BAR_PACKAGE)) + } + + @Test + fun `inclusion filter should override exclusion filter`() { + val filter = createTestCasesFilter(listOf("notPackage foo", "class whatever.Bar")) + + assertFalse(filter(FOO_PACKAGE)) + assertFalse(filter(BAR_PACKAGE)) + assertTrue(filter(BAR_CLASSNAME)) + } + + @Test + fun `withClassName should correctly filter classes with similar name`() { + // test-targets: + // - class foo.bar.Class1 + // should filter foo.bar.Class11, foo.bar.Class101 ... + // the same is applicable for methods + + val filter = createTestCasesFilter( + listOf( + "class anyPackage_1.anyClass_1", + "class anyPackage_3.anyClass_3#anyMethod_3" + ) + ) + + val tests = listOf( + TargetsHelper(pack = "anyPackage_1", cl = "anyClass_1", m = "anyMethod_1", annotation = "Foo"), + TargetsHelper(pack = "anyPackage_1", cl = "anyClass_1", m = "anyMethod_2", annotation = "Foo"), + TargetsHelper(pack = "anyPackage_1", cl = "anyClass_12", m = "anyMethod_1", annotation = "Bar"), + TargetsHelper(pack = "anyPackage_1", cl = "anyClass_12", m = "anyMethod_12", annotation = "Bar"), + TargetsHelper(pack = "anyPackage_3", cl = "anyClass_3", m = "anyMethod_3", annotation = "Bar"), + TargetsHelper(pack = "anyPackage_3", cl = "anyClass_3", m = "anyMethod_32", annotation = "Bar"), + TargetsHelper(pack = "anyPackage_3", cl = "anyClass_32", m = "anyMethod_3", annotation = "Bar"), + TargetsHelper(pack = "anyPackage_3", cl = "anyClass_32", m = "anyMethod_32", annotation = "Bar") + ).map { getDefaultTestMethod(it.fullView, it.annotation) } + + val expected = listOf( + "anyPackage_1.anyClass_1#anyMethod_1", + "anyPackage_1.anyClass_1#anyMethod_2", + "anyPackage_3.anyClass_3#anyMethod_3" + ) + + val result = tests.withFilter(filter) + + assertEquals(expected, result) + } +} + +private fun getPath(path: String): Path = + Paths.get(path).toAbsolutePath().normalize() + +private fun getDefaultTestMethod(testName: String, annotation: String): Target = + testName to listOf(annotation) + +private fun List.add(method: Target) = listOf(*this.toTypedArray() + method) + +private fun List.withFilter(shouldRun: ShouldRun, enrich: Boolean = true) = + if (enrich) add(getDefaultTestMethod("should.be#filtered", "AnyAnnotation")) + .filter(shouldRun).map { it.first } + else this + .filter(shouldRun).map { it.first } + +private class TargetsHelper( + private val pack: String, + private val cl: String, + private val m: String, + val annotation: String +) { + val classView: String + get() = "class $pack.$cl" + + val packageView: String + get() = "package $pack" + + val methodView: String + get() = "class $fullView" + + val fullView: String + get() = "$pack.$cl#$m" +} diff --git a/tool/filter/src/test/resources/dummy-tests-file.txt b/tool/filter/src/test/resources/dummy-tests-file.txt new file mode 100644 index 0000000000..3bd1f0e297 --- /dev/null +++ b/tool/filter/src/test/resources/dummy-tests-file.txt @@ -0,0 +1,2 @@ +foo +bar diff --git a/tool/filter/src/test/resources/exclude-tests.txt b/tool/filter/src/test/resources/exclude-tests.txt new file mode 100644 index 0000000000..6c1104f018 --- /dev/null +++ b/tool/filter/src/test/resources/exclude-tests.txt @@ -0,0 +1,2 @@ +foo +whatever.Bar From 3b70c67a86aeba8e9ead7d9e3df79d2bde33bdf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 30 Jun 2021 23:56:35 +0200 Subject: [PATCH 2/4] Fix lint error --- .../src/main/kotlin/flank/filter/internal/{Data.kt => Test.kt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tool/filter/src/main/kotlin/flank/filter/internal/{Data.kt => Test.kt} (100%) diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/Data.kt b/tool/filter/src/main/kotlin/flank/filter/internal/Test.kt similarity index 100% rename from tool/filter/src/main/kotlin/flank/filter/internal/Data.kt rename to tool/filter/src/main/kotlin/flank/filter/internal/Test.kt From ce88ce584503032adfa6a269e06a6cdc393bcf32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 1 Jul 2021 00:34:10 +0200 Subject: [PATCH 3/4] Fix lint issue --- tool/filter/src/test/kotlin/flank/filter/FilterKtTest.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/tool/filter/src/test/kotlin/flank/filter/FilterKtTest.kt b/tool/filter/src/test/kotlin/flank/filter/FilterKtTest.kt index f5d5296ab9..8ab84a8c7b 100644 --- a/tool/filter/src/test/kotlin/flank/filter/FilterKtTest.kt +++ b/tool/filter/src/test/kotlin/flank/filter/FilterKtTest.kt @@ -1,7 +1,6 @@ package flank.filter import flank.filter.internal.fromTestTargets - import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue From 669abdf5697d5ef1c54c11415ef532ea95e6c94e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 1 Jul 2021 17:25:06 +0200 Subject: [PATCH 4/4] Simplify names --- .../filter/internal/FilterFromTestTarget.kt | 18 +++++++++--------- .../main/kotlin/flank/filter/internal/Test.kt | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/FilterFromTestTarget.kt b/tool/filter/src/main/kotlin/flank/filter/internal/FilterFromTestTarget.kt index 3ac24988a8..70ac01eaa1 100644 --- a/tool/filter/src/main/kotlin/flank/filter/internal/FilterFromTestTarget.kt +++ b/tool/filter/src/main/kotlin/flank/filter/internal/FilterFromTestTarget.kt @@ -36,15 +36,15 @@ private fun parseTarget(target: String): Target = private fun Target.createFilter(): Test.Filter = when (type) { - Test.Target.Type.ARGUMENT_TEST_CLASS -> withClassName(args) - Test.Target.Type.ARGUMENT_NOT_TEST_CLASS -> not(withClassName(args)) - Test.Target.Type.ARGUMENT_TEST_PACKAGE -> withPackageName(args) - Test.Target.Type.ARGUMENT_NOT_TEST_PACKAGE -> not(withPackageName(args)) - Test.Target.Type.ARGUMENT_ANNOTATION -> withAnnotation(args) - Test.Target.Type.ARGUMENT_NOT_ANNOTATION -> not(withAnnotation(args)) - Test.Target.Type.ARGUMENT_TEST_FILE -> fromTestFile(args) - Test.Target.Type.ARGUMENT_NOT_TEST_FILE -> not(fromTestFile(args)) - Test.Target.Type.ARGUMENT_TEST_SIZE -> withSize(args) + Test.Target.Type.TEST_CLASS -> withClassName(args) + Test.Target.Type.NOT_TEST_CLASS -> not(withClassName(args)) + Test.Target.Type.TEST_PACKAGE -> withPackageName(args) + Test.Target.Type.NOT_TEST_PACKAGE -> not(withPackageName(args)) + Test.Target.Type.ANNOTATION -> withAnnotation(args) + Test.Target.Type.NOT_ANNOTATION -> not(withAnnotation(args)) + Test.Target.Type.TEST_FILE -> fromTestFile(args) + Test.Target.Type.NOT_TEST_FILE -> not(fromTestFile(args)) + Test.Target.Type.TEST_SIZE -> withSize(args) else -> throw IllegalArgumentException("Filtering option $type not supported") } diff --git a/tool/filter/src/main/kotlin/flank/filter/internal/Test.kt b/tool/filter/src/main/kotlin/flank/filter/internal/Test.kt index 45cfd12ef7..4a843753c9 100644 --- a/tool/filter/src/main/kotlin/flank/filter/internal/Test.kt +++ b/tool/filter/src/main/kotlin/flank/filter/internal/Test.kt @@ -28,19 +28,19 @@ internal object Test { internal object Target { object Type { - const val ARGUMENT_TEST_CLASS = "class" - const val ARGUMENT_NOT_TEST_CLASS = "notClass" + const val TEST_CLASS = "class" + const val NOT_TEST_CLASS = "notClass" - const val ARGUMENT_TEST_SIZE = "size" + const val TEST_SIZE = "size" - const val ARGUMENT_ANNOTATION = "annotation" - const val ARGUMENT_NOT_ANNOTATION = "notAnnotation" + const val ANNOTATION = "annotation" + const val NOT_ANNOTATION = "notAnnotation" - const val ARGUMENT_TEST_PACKAGE = "package" - const val ARGUMENT_NOT_TEST_PACKAGE = "notPackage" + const val TEST_PACKAGE = "package" + const val NOT_TEST_PACKAGE = "notPackage" - const val ARGUMENT_TEST_FILE = "testFile" - const val ARGUMENT_NOT_TEST_FILE = "notTestFile" + const val TEST_FILE = "testFile" + const val NOT_TEST_FILE = "notTestFile" } object Size {