From e5d362f4dad35a66a08e6903543fe42da08b7dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 3 Jun 2021 05:06:23 +0200 Subject: [PATCH 01/29] Add parallel execution module --- settings.gradle.kts | 2 + tool/execution/parallel/build.gradle.kts | 18 +++ .../flank/exection/parallel/Parallel.kt | 102 ++++++++++++++++ .../exection/parallel/internal/Execute.kt | 64 ++++++++++ .../flank/exection/parallel/internal/Utils.kt | 16 +++ .../flank/exection/parallel/ParallelTest.kt | 32 +++++ .../parallel/example/ExampleExecution.kt | 111 ++++++++++++++++++ tool/execution/synchronized/build.gradle.kts | 18 +++ 8 files changed, 363 insertions(+) create mode 100644 tool/execution/parallel/build.gradle.kts create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execute.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt create mode 100644 tool/execution/synchronized/build.gradle.kts diff --git a/settings.gradle.kts b/settings.gradle.kts index f456bc7bf9..2e616e754d 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -31,6 +31,8 @@ include( ":tool:junit", ":tool:log", ":tool:log:format", + ":tool:execution:parallel", + ":tool:execution:synchronized", ) plugins { diff --git a/tool/execution/parallel/build.gradle.kts b/tool/execution/parallel/build.gradle.kts new file mode 100644 index 0000000000..93570d09cd --- /dev/null +++ b/tool/execution/parallel/build.gradle.kts @@ -0,0 +1,18 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin(Plugins.Kotlin.PLUGIN_JVM) + kotlin(Plugins.Kotlin.PLUGIN_SERIALIZATION) version Versions.KOTLIN +} + +repositories { + mavenCentral() + maven(url = "https://kotlin.bintray.com/kotlinx") +} + +tasks.withType { kotlinOptions.jvmTarget = "1.8" } + +dependencies { + implementation(Dependencies.KOTLIN_COROUTINES_CORE) + testImplementation(Dependencies.JUNIT) +} diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt new file mode 100644 index 0000000000..e174735eca --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -0,0 +1,102 @@ +package flank.exection.parallel + +import flank.exection.parallel.internal.CreateFunction +import flank.exection.parallel.internal.execute +import flank.exection.parallel.internal.lazy +import java.lang.System.currentTimeMillis + +object Parallel { + + interface Context { + val out: Output + } + + /** + * Abstraction for execution data provider which is also an context for step execution. + * For initialization purpose some properties are exposed as variable + */ + abstract class Store { + /** + * The state properties map. Is for initialization purpose, and shouldn't be mutated after initialization + */ + internal lateinit var state: ParallelState + + /** + * Accessor for overridden context output + */ + val out: Output by OutputType() + + /** + * Factory method for creating properties from data objects. + */ + operator fun Type.invoke(): Lazy = lazy(this) + } + + /** + * Common interface for the tasks arguments and return values. + */ + interface Type + + internal object OutputType : Type + + /** + * Structure that is representing single task of execution. + */ + data class Task( + val signature: Signature, + val execute: C.() -> suspend ParallelState.() -> R + ) { + + /** + * The step signature that contains arguments types and returned type. + */ + class Signature( + val args: Set>, + val returns: Type, + ) + } + + /** + * Parameterized factory for creating step functions in scope of [S] + */ + class Function(override val store: () -> S) : CreateFunction + + data class Event( + val type: Type<*>, + val data: Any, + val timestamp: Long = currentTimeMillis(), + ) { + object Start + object Stop + } +} + +typealias Output = Any.() -> Unit + +typealias ParallelState = Map, Any> + +typealias Tasks = Set> + +/** + * Factory function for creating [Parallel.Task.Signature] from expected type and argument types. + */ +infix fun Parallel.Type.from(args: Set>) = + Parallel.Task.Signature(args, this) + +/** + * Factory function for creating [Parallel.Task] from signature and execute function + */ +infix fun Parallel.Task.Signature.using(execute: C.() -> suspend ParallelState.() -> R) = + Parallel.Task(this, execute) + +/** + * Factory function for creating [Parallel.Task] directly from expected type without specifying required arguments. + */ +infix fun Parallel.Type.using(execute: C.() -> suspend ParallelState.() -> R) = + Parallel.Task.Signature(emptySet(), this).using(execute) + +/** + * Execute tasks on a given state + */ +suspend fun C.execute(tasks: Tasks, state: ParallelState = emptyMap()) = + execute(context = this, state, tasks) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execute.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execute.kt new file mode 100644 index 0000000000..3e18ff50b9 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execute.kt @@ -0,0 +1,64 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Output +import flank.exection.parallel.Parallel.Context +import flank.exection.parallel.Parallel.Event +import flank.exection.parallel.Parallel.OutputType +import flank.exection.parallel.Parallel.Task +import flank.exection.parallel.Parallel.Type +import flank.exection.parallel.ParallelState +import flank.exection.parallel.Tasks +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.flow.consumeAsFlow +import kotlinx.coroutines.flow.last +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.scan +import kotlinx.coroutines.launch +import java.util.concurrent.ConcurrentHashMap + +internal suspend fun execute( + context: C, + initialState: ParallelState, + tasks: Tasks, +): ParallelState = coroutineScope { + val jobs = mutableMapOf, Job>() + val remaining = ConcurrentHashMap(tasks.groupBy { step -> step.signature.args }) + val channel = Channel>(Channel.BUFFERED) + val returnSignatures = tasks.map { it.signature.returns } + channel.consumeAsFlow().scan(initialState, merge).onEach { state -> + if (state.keys.containsAll(returnSignatures)) channel.close() + remaining.filterKeys(state.keys::containsAll).apply { remaining -= keys } + .values.flatten().forEach { task: Task -> + val type: Type = task.signature.returns + jobs += type to launch(Dispatchers.IO) { + val out: Output = { context.out(Event(type, this)) } + try { + Event.Start.out() + val result = task.execute(context)(state + (OutputType to out)) + Event.Stop.out() + channel.trySend(type to result) + } catch (throwable: Throwable) { + throwable.out() + channel.trySend(type to throwable) + val queued = remaining.toMap().also { remaining.clear() } + jobs.minus(type).filter { it.value.isActive }.forEach { (type, job) -> + job.cancel("aborted") + channel.trySend(type to job) + } + queued.values.flatten().forEach { + channel.trySend(it.signature.returns to it) + } + } + } + } + }.last() +} + +private val merge: suspend (ParallelState, Value) -> Map, Any> = + { state, prop -> state + prop } + +private typealias Value = Pair, Any> diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt new file mode 100644 index 0000000000..fc95c0a214 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt @@ -0,0 +1,16 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Parallel +import flank.exection.parallel.ParallelState + +@Suppress("UNCHECKED_CAST") +internal fun Parallel.Store.lazy( + type: Parallel.Type +): Lazy = lazy { state[type] as T } + +internal interface CreateFunction { + val store: () -> S + operator fun invoke(create: suspend S.() -> T): suspend ParallelState.() -> T = { + store().also { it.state = this }.run { create() } + } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt new file mode 100644 index 0000000000..9c2561cd76 --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt @@ -0,0 +1,32 @@ +package flank.exection.parallel + +import flank.exection.parallel.example.ExampleExecution +import flank.exection.parallel.example.invoke +import org.junit.Test +import java.text.SimpleDateFormat + +class ParallelTest { + + @Test + fun execute() { + val result = object : ExampleExecution.Context { + override val args = ExampleExecution.Args( + wait = 50, + a = 5, + b = 8, + c = 13, + ) + override val out: Output = { + when (this) { + is Parallel.Event -> println("${format.format(timestamp)} - ${type::class.simpleName}: $data") + else -> Unit + } + } + private val format = SimpleDateFormat("yyyy.MM.dd HH:mm:ss.SSS") + }.invoke() + + result.forEach { + println(it) + } + } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt new file mode 100644 index 0000000000..14354ac70d --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt @@ -0,0 +1,111 @@ +package flank.exection.parallel.example + +import flank.exection.parallel.Parallel +import flank.exection.parallel.Parallel.Type +import flank.exection.parallel.example.ExampleExecution.A +import flank.exection.parallel.example.ExampleExecution.B +import flank.exection.parallel.example.ExampleExecution.C +import flank.exection.parallel.example.ExampleExecution.Context +import flank.exection.parallel.example.ExampleExecution.Failing +import flank.exection.parallel.example.ExampleExecution.Summary +import flank.exection.parallel.example.ExampleExecution.func +import flank.exection.parallel.execute +import flank.exection.parallel.from +import flank.exection.parallel.using +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking + +// ======================= Public API ======================= + +object ExampleExecution { + + /** + * Context for [ExampleExecution] + */ + interface Context : Parallel.Context { + val args: Args + } + + /** + * Arguments for [ExampleExecution] + */ + data class Args( + val wait: Long, + val a: Int, + val b: Int, + val c: Int + ) + + // Type definitions for [ExampleExecution] + object A : Type> + object B : Type> + object C : Type> + object Failing : Type + object Summary : Type> + + /** + * Wrapper class for state values with exposed static accessors. + */ + class Store : Parallel.Store() { + val a by A() + val b by B() + val c by C() + } + + /** + * Factory method for creating step functions in scope of [Store]. + */ + internal val func = Parallel.Function(::Store) + + /** + * List of tasks available for [ExampleExecution] + */ + internal val tasks = setOf( + errorAfterA, + produceA, + produceB, + produceC, + groupAndCount, + ) +} + +fun Context.invoke() = runBlocking { execute(ExampleExecution.tasks) } + +// ======================= Internal Tasks ======================= + +internal val errorAfterA = Failing from setOf(A) using fun Context.() = func { + throw IllegalStateException(args.toString()) +} + +internal val produceA = A using fun Context.() = func { + (0..args.a).asFlow().onEach { + out("A" to it) + delay((0..args.wait).random()) + }.toList() +} + +internal val produceB = B using fun Context.() = func { + (0..args.b).asFlow().onEach { + out("B" to it) + delay((0..args.wait).random()) + }.toList() +} + +internal val produceC = C using fun Context.() = func { + (0..args.c).asFlow().onEach { + out("C" to it) + delay((0..args.wait).random()) + }.toList() +} + +internal val groupAndCount = Summary from setOf(A, B, C) using fun Context.() = func { + delay(args.wait) + (a + b + c) + .groupBy { it } + .mapValues { (_, values) -> values.count() } +} + + diff --git a/tool/execution/synchronized/build.gradle.kts b/tool/execution/synchronized/build.gradle.kts new file mode 100644 index 0000000000..93570d09cd --- /dev/null +++ b/tool/execution/synchronized/build.gradle.kts @@ -0,0 +1,18 @@ +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + kotlin(Plugins.Kotlin.PLUGIN_JVM) + kotlin(Plugins.Kotlin.PLUGIN_SERIALIZATION) version Versions.KOTLIN +} + +repositories { + mavenCentral() + maven(url = "https://kotlin.bintray.com/kotlinx") +} + +tasks.withType { kotlinOptions.jvmTarget = "1.8" } + +dependencies { + implementation(Dependencies.KOTLIN_COROUTINES_CORE) + testImplementation(Dependencies.JUNIT) +} From 9d649ee13a5ba15446de49a1dfd01d9e8798a3ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 3 Jun 2021 05:57:27 +0200 Subject: [PATCH 02/29] Add subgraph selection --- .../flank/exection/parallel/Parallel.kt | 5 ++-- .../flank/exection/parallel/internal/Utils.kt | 20 ++++++++++++++++ .../flank/exection/parallel/ParallelTest.kt | 4 +--- .../parallel/example/ExampleExecution.kt | 2 +- .../exection/parallel/internal/UtilsKtTest.kt | 23 +++++++++++++++++++ 5 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/UtilsKtTest.kt diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt index e174735eca..45c5289233 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -3,6 +3,7 @@ package flank.exection.parallel import flank.exection.parallel.internal.CreateFunction import flank.exection.parallel.internal.execute import flank.exection.parallel.internal.lazy +import flank.exection.parallel.internal.subgraph import java.lang.System.currentTimeMillis object Parallel { @@ -98,5 +99,5 @@ infix fun Parallel.Type.using(execute: C.() - /** * Execute tasks on a given state */ -suspend fun C.execute(tasks: Tasks, state: ParallelState = emptyMap()) = - execute(context = this, state, tasks) +suspend fun C.execute(tasks: Tasks, select: Tasks = emptySet(), state: ParallelState = emptyMap()) = + execute(context = this, state, tasks.subgraph(select)) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt index fc95c0a214..af9c9235fe 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt @@ -2,6 +2,7 @@ package flank.exection.parallel.internal import flank.exection.parallel.Parallel import flank.exection.parallel.ParallelState +import flank.exection.parallel.Tasks @Suppress("UNCHECKED_CAST") internal fun Parallel.Store.lazy( @@ -14,3 +15,22 @@ internal interface CreateFunction { store().also { it.state = this }.run { create() } } } + +tailrec fun Tasks.subgraph( + current: Tasks, + acc: Tasks = + if (current.isEmpty()) this + else emptySet(), +): Tasks = when { + current.isEmpty() -> acc + else -> subgraph( + current = current.flatMap { it.signature.args } + .mapNotNull(this::findByReturn) + .toSet(), + acc = acc + current + ) +} + +fun Tasks.findByReturn( + type: Parallel.Type<*> +) = find { it.signature.returns == type } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt index 9c2561cd76..cafb53adaf 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt @@ -25,8 +25,6 @@ class ParallelTest { private val format = SimpleDateFormat("yyyy.MM.dd HH:mm:ss.SSS") }.invoke() - result.forEach { - println(it) - } + result.forEach { println(it) } } } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt index 14354ac70d..930aad19a1 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt @@ -72,7 +72,7 @@ object ExampleExecution { ) } -fun Context.invoke() = runBlocking { execute(ExampleExecution.tasks) } +fun Context.invoke() = runBlocking { execute(ExampleExecution.tasks, setOf(groupAndCount)) } // ======================= Internal Tasks ======================= diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/UtilsKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/UtilsKtTest.kt new file mode 100644 index 0000000000..575428f205 --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/UtilsKtTest.kt @@ -0,0 +1,23 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Tasks +import flank.exection.parallel.example.ExampleExecution +import flank.exection.parallel.example.errorAfterA +import flank.exection.parallel.example.produceA +import org.junit.Assert +import org.junit.Test + +class UtilsKtTest { + + @Test + fun find() { + val expected: Tasks = setOf( + errorAfterA, + produceA, + ) + val actual = ExampleExecution.tasks.subgraph(setOf(errorAfterA)) + + println(actual.map { it.signature.returns.javaClass.simpleName }) + Assert.assertEquals(expected, actual) + } +} From c952d1d199600acdc50b71bff573e80cdc56588a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 7 Jun 2021 07:56:34 +0200 Subject: [PATCH 03/29] Update --- docs/hld/parallel-execution.puml | 65 ++++++ .../kotlin/flank/exection/parallel/Ext.kt | 15 ++ .../kotlin/flank/exection/parallel/Factory.kt | 25 +++ .../flank/exection/parallel/Parallel.kt | 94 +++++---- .../exection/parallel/internal/Execute.kt | 64 ------ .../exection/parallel/internal/Execution.kt | 191 ++++++++++++++++++ .../exection/parallel/internal/Reduce.kt | 38 ++++ .../flank/exection/parallel/internal/Utils.kt | 37 +--- .../exection/parallel/internal/Validate.kt | 24 +++ .../kotlin/flank/exection/parallel/Example.kt | 133 ++++++++++++ .../flank/exection/parallel/ParallelTest.kt | 82 ++++++-- .../parallel/example/ExampleExecution.kt | 111 ---------- .../parallel/internal/ReduceKtTest.kt | 44 ++++ .../exection/parallel/internal/UtilsKtTest.kt | 23 --- .../parallel/internal/ValidateKtTest.kt | 46 +++++ 15 files changed, 702 insertions(+), 290 deletions(-) create mode 100644 docs/hld/parallel-execution.puml create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt delete mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execute.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt delete mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt delete mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/UtilsKtTest.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt diff --git a/docs/hld/parallel-execution.puml b/docs/hld/parallel-execution.puml new file mode 100644 index 0000000000..11080238d5 --- /dev/null +++ b/docs/hld/parallel-execution.puml @@ -0,0 +1,65 @@ +@startuml +skinparam ConditionEndStyle hline + +|Invoke execution| +split +-[hidden]-> +start +:arguments; +:initial state; +repeat :Handle state} +-[dotted]-> +if (is channel closed for receive?) then +stop +else (no) +endif +:next result< +if (on first error) then (true) +:Abort execution} +else (false) +endif +repeat while (accumulate result) +detach + +|Handle state| +split again +-[hidden]-> +if (is complete) then (yes) +:close channel/ +elseif (is finished) then (yes) +:nothing to do; +detach +else +:filter tasks for current state; +:group them by arguments; +repeat :prepare properties| +repeat :Execute task} +repeat while (for each task) +repeat while (for each group) +endif +detach + +|Execute task| +split again +-[hidden]-> +partition async { +:log started; +:prepare context; +if (execute task in context) +:log succeed; +else +:log failed; +endif +:send result] +detach +} + +|Abort execution| +split again +-[hidden]-> +:cancel scope; +repeat :send "not started"] +repeat while (for each remaining task) +:close channel/ +detach +@enduml diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt new file mode 100644 index 0000000000..50b4ae1740 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt @@ -0,0 +1,15 @@ +package flank.exection.parallel + +/** + * Extension for executing tasks on a given state. + */ +operator fun Tasks.invoke( + vararg state: Pair, Any> +) = invoke(state.toMap()) + +/** + * Extension for reducing tasks to returned with dependencies. + */ +operator fun Tasks.invoke( + vararg returns: Parallel.Type<*> +) = invoke(returns.toSet()) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt new file mode 100644 index 0000000000..dc59332dc8 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt @@ -0,0 +1,25 @@ +package flank.exection.parallel + +/** + * Factory function for creating [Parallel.Task.Signature] from expected type and argument types. + */ +infix fun Parallel.Type.from( + args: Set> +): Parallel.Task.Signature = + Parallel.Task.Signature(this, args) + +/** + * Factory function for creating [Parallel.Task] from signature and execute function. + */ +infix fun Parallel.Task.Signature.using( + execute: suspend ParallelState.() -> R +): Parallel.Task = + Parallel.Task(this, execute) + +/** + * Factory function for creating [Parallel.Task] from expected type and execute function without specifying required arguments. + */ +infix fun Parallel.Type.using( + execute: suspend ParallelState.() -> R +): Parallel.Task = + from(emptySet()).using(execute) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt index 45c5289233..1ca890ab70 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -1,36 +1,49 @@ package flank.exection.parallel -import flank.exection.parallel.internal.CreateFunction -import flank.exection.parallel.internal.execute +import flank.exection.parallel.internal.CreateTaskFunction +import flank.exection.parallel.internal.Execution +import flank.exection.parallel.internal.invoke import flank.exection.parallel.internal.lazy -import flank.exection.parallel.internal.subgraph +import flank.exection.parallel.internal.reduce +import flank.exection.parallel.internal.validate import java.lang.System.currentTimeMillis -object Parallel { +/** + * Reduce given [Tasks] by [returns] types to remove unneeded tasks from the graph. + * The returned graph will contain only tasks that are returning selected types, their dependencies and derived dependencies. + */ +operator fun Tasks.invoke(returns: Set>): Tasks = reduce(returns) - interface Context { - val out: Output - } +/** + * Execute tasks in parallel with a given args. + * Before executing the [validate] is performed on a [Tasks] for a given [args] to detect missing dependencies. + */ +infix operator fun Tasks.invoke(args: ParallelState) = Execution(validate(args), args).invoke() + +/** + * The namespace for parallel execution primitives. + */ +object Parallel { /** - * Abstraction for execution data provider which is also an context for step execution. - * For initialization purpose some properties are exposed as variable + * Abstraction for execution data provider which is also an context for task execution. + * For initialization purpose some properties are exposed as variable. */ - abstract class Store { + open class Context { /** - * The state properties map. Is for initialization purpose, and shouldn't be mutated after initialization + * The state properties map. Is for initialization purpose, and shouldn't be mutated after initialization. */ internal lateinit var state: ParallelState /** - * Accessor for overridden context output + * Exposed reference to output, accessible by tasks. */ - val out: Output by OutputType() + val out: Output by Logger() /** * Factory method for creating properties from data objects. */ - operator fun Type.invoke(): Lazy = lazy(this) + operator fun Type.invoke(): Lazy = lazy(type = this) } /** @@ -38,29 +51,26 @@ object Parallel { */ interface Type - internal object OutputType : Type - /** * Structure that is representing single task of execution. */ - data class Task( + data class Task( val signature: Signature, - val execute: C.() -> suspend ParallelState.() -> R + val execute: suspend ParallelState.() -> R ) { - /** - * The step signature that contains arguments types and returned type. + * The task signature that contains arguments types and returned type. */ - class Signature( - val args: Set>, + data class Signature( val returns: Type, + val args: Set>, ) - } - /** - * Parameterized factory for creating step functions in scope of [S] - */ - class Function(override val store: () -> S) : CreateFunction + /** + * Parameterized factory for creating task functions in scope of [X]. + */ + class Body(override val context: () -> X) : CreateTaskFunction + } data class Event( val type: Type<*>, @@ -70,34 +80,30 @@ object Parallel { object Start object Stop } -} -typealias Output = Any.() -> Unit + object Logger : Type -typealias ParallelState = Map, Any> - -typealias Tasks = Set> + data class ValidationError( + val data: MissingDependencies + ) : Exception("Cannot resolve dependencies $data") +} /** - * Factory function for creating [Parallel.Task.Signature] from expected type and argument types. + * Common signature for structural log output. */ -infix fun Parallel.Type.from(args: Set>) = - Parallel.Task.Signature(args, this) +internal typealias Output = Any.() -> Unit /** - * Factory function for creating [Parallel.Task] from signature and execute function + * Immutable state for parallel execution. */ -infix fun Parallel.Task.Signature.using(execute: C.() -> suspend ParallelState.() -> R) = - Parallel.Task(this, execute) +internal typealias ParallelState = Map, Any> /** - * Factory function for creating [Parallel.Task] directly from expected type without specifying required arguments. + * Type for group of parallel tasks. Each task must be unique in group. */ -infix fun Parallel.Type.using(execute: C.() -> suspend ParallelState.() -> R) = - Parallel.Task.Signature(emptySet(), this).using(execute) +internal typealias Tasks = Set> /** - * Execute tasks on a given state + * The [Map.values] are representing missing dependencies required by the tasks to provide [Map.keys]. */ -suspend fun C.execute(tasks: Tasks, select: Tasks = emptySet(), state: ParallelState = emptyMap()) = - execute(context = this, state, tasks.subgraph(select)) +internal typealias MissingDependencies = Map, Set>> diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execute.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execute.kt deleted file mode 100644 index 3e18ff50b9..0000000000 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execute.kt +++ /dev/null @@ -1,64 +0,0 @@ -package flank.exection.parallel.internal - -import flank.exection.parallel.Output -import flank.exection.parallel.Parallel.Context -import flank.exection.parallel.Parallel.Event -import flank.exection.parallel.Parallel.OutputType -import flank.exection.parallel.Parallel.Task -import flank.exection.parallel.Parallel.Type -import flank.exection.parallel.ParallelState -import flank.exection.parallel.Tasks -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.Job -import kotlinx.coroutines.cancel -import kotlinx.coroutines.channels.Channel -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.flow.consumeAsFlow -import kotlinx.coroutines.flow.last -import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.flow.scan -import kotlinx.coroutines.launch -import java.util.concurrent.ConcurrentHashMap - -internal suspend fun execute( - context: C, - initialState: ParallelState, - tasks: Tasks, -): ParallelState = coroutineScope { - val jobs = mutableMapOf, Job>() - val remaining = ConcurrentHashMap(tasks.groupBy { step -> step.signature.args }) - val channel = Channel>(Channel.BUFFERED) - val returnSignatures = tasks.map { it.signature.returns } - channel.consumeAsFlow().scan(initialState, merge).onEach { state -> - if (state.keys.containsAll(returnSignatures)) channel.close() - remaining.filterKeys(state.keys::containsAll).apply { remaining -= keys } - .values.flatten().forEach { task: Task -> - val type: Type = task.signature.returns - jobs += type to launch(Dispatchers.IO) { - val out: Output = { context.out(Event(type, this)) } - try { - Event.Start.out() - val result = task.execute(context)(state + (OutputType to out)) - Event.Stop.out() - channel.trySend(type to result) - } catch (throwable: Throwable) { - throwable.out() - channel.trySend(type to throwable) - val queued = remaining.toMap().also { remaining.clear() } - jobs.minus(type).filter { it.value.isActive }.forEach { (type, job) -> - job.cancel("aborted") - channel.trySend(type to job) - } - queued.values.flatten().forEach { - channel.trySend(it.signature.returns to it) - } - } - } - } - }.last() -} - -private val merge: suspend (ParallelState, Value) -> Map, Any> = - { state, prop -> state + prop } - -private typealias Value = Pair, Any> diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt new file mode 100644 index 0000000000..61238939d2 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt @@ -0,0 +1,191 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Output +import flank.exection.parallel.Parallel.Event +import flank.exection.parallel.Parallel.Logger +import flank.exection.parallel.Parallel.Task +import flank.exection.parallel.Parallel.Type +import flank.exection.parallel.ParallelState +import flank.exection.parallel.Tasks +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.consumeAsFlow +import kotlinx.coroutines.flow.onCompletion +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.scan +import kotlinx.coroutines.launch + +/** + * Invoke the given [Execution] in parallel. + */ +internal operator fun Execution.invoke(): Flow = + channel.consumeAsFlow() + + // Abort the execution if any task was failed and returned the throwable value. + .onEach { (type, value) -> if (value is Throwable && isNotClosed()) abortBy(type, value) } + + // Accumulate each received value in state. + .scan(initial) { state, value -> state + value } + + // Handle state changes. + .onEach { state: ParallelState -> + when { + // State is containing all required types. Close the channel which will finish the execution. + isComplete(state) -> channel.close() + + // The execution is already finished and just collecting the rest of properties. Nothing to do here. + isFinished(state) -> Unit + + // For each task group that is satisfied by current state. + else -> filterTasksFor(state).forEach { (args, tasks) -> + + // Prepare common properties for current iteration. + val properties = state.filterKeys(args::contains) + + // Execute tasks with required properties. + tasks.forEach { task: Task<*> -> execute(task, properties) } + } + } + } + + // Cancel the coroutine scope for tasks. + .onCompletion { scope.cancel() } + +/** + * Internal context of the tasks execution. + */ +internal class Execution( + /** + * List of tasks supposed to run in parallel. + */ + tasks: Tasks, + /** + * The initial state required to satisfy the tasks. + */ + val initial: ParallelState, +) { + /** + * Coroutine scope for executing tasks. + */ + val scope = CoroutineScope(Job() + Dispatchers.IO) + + /** + * Map of jobs (tasks) that are resolving values for associated types. + */ + val jobs = mutableMapOf, Job>() + + /** + * Channel for receiving resolved values from tasks. + */ + val channel = Channel, Any>>(Channel.BUFFERED) + + /** + * The set of value types required to complete the execution. + */ + val required = tasks.map { task -> task.signature.returns }.toSet() + + /** + * Map of remaining tasks for run grouped by arguments. + */ + val remaining = tasks.groupBy { task -> task.signature.args }.toMutableMap() + + /** + * Reference to optional output for structural logs. + */ + @Suppress("UNCHECKED_CAST") + val output = initial[Logger] as? Output +} + +// ========================== Private functions ========================== + +/** + * The execution is not closed if the properties channel is not closed for send. + * Other words, task can still send property to update the state. + */ +private fun Execution.isNotClosed(): Boolean = !channel.isClosedForSend + +/** + * Cancel running jobs, update state for cancellation values and close the execution. + */ +private fun Execution.abortBy(type: Type<*>, cause: Throwable) { + + // Abort running jobs. + scope.cancel("aborted by ${type.javaClass.name}", cause) + + // Add remaining tasks to state. + remaining.toMap().values.flatten().forEach { task -> + channel.trySend(task.signature.returns to task) + } + + // Close execution channel. + channel.close() +} + +/** + * The execution is complete when all required types was accumulated to state. + */ +private fun Execution.isComplete(state: ParallelState): Boolean = + state.keys.containsAll(required) + +/** + * Execution is finished when any value is [Throwable] or there are no remaining [Task] to run. + */ +private fun Execution.isFinished(state: ParallelState): Boolean = + state.any { (_, value) -> value is Throwable } || remaining.isEmpty() + +/** + * Get tasks available to run in the current state. + * + * @throws IllegalStateException If there are no tasks to run and no running jobs for a given state. + */ +private fun Execution.filterTasksFor(state: ParallelState): Map>, List>> = + remaining.filterKeys(state.keys::containsAll) + + // Remove from queue the tasks for current iteration. + .apply { remaining -= keys } + + // Check if execution is not detached. Typically will fail if broken graph was not validated before execution and passed here. + .apply { check(isNotEmpty() || jobs.any { (_, job) -> job.isActive }) { "Deadlock!" } } + +/** + * Execute given tasks as coroutine job. + */ +private fun Execution.execute( + task: Task<*>, + args: Map, Any>, +) { + // Obtain return type for current task. + val type: Type<*> = task.signature.returns + + // Keep task job. + jobs += type to scope.launch { + + // Extend root output for adding additional data. + val out: Output = output?.let { { it(Event(type, this)) } } ?: {} + + // Log the task was started + Event.Start.out() + + // Try to execute tasks. + val (result, eventFinish) = try { + + // Prepare context for task + val context = initial + args + (Logger to out) + + // Execute task on given state + task.execute(context) to Event.Stop + } catch (throwable: Throwable) { + throwable to throwable + } + + // Log the task was finished. + eventFinish.out() + + // Feed up the execution. + channel.trySend(type to result) + } +} diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt new file mode 100644 index 0000000000..4cc3897d68 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt @@ -0,0 +1,38 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Parallel +import flank.exection.parallel.Tasks + +internal infix fun Tasks.reduce( + select: Set> +): Tasks = + filter { task -> task.signature.returns in select } + .toSet() + .reduce(this) + +/** + * Reduce [all] steps to given receiver steps and their dependencies. + * + * @receiver The task selector for current reducing step. + * @param all The list of all tasks that are under reducing. + * @param acc Currently accumulated tasks. + * @return Accumulated tasks if selector is empty. + */ +private tailrec fun Tasks.reduce( + all: Tasks, + acc: Tasks = + if (isEmpty()) all + else emptySet(), +): Tasks = + when { + isEmpty() -> acc + else -> flatMap { task -> task.signature.args } + .mapNotNull(all::findByReturnType) + .toSet() + .reduce(all, acc + this) + } + +private fun Tasks.findByReturnType( + type: Parallel.Type<*> +): Parallel.Task<*>? = + find { task -> task.signature.returns == type } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt index af9c9235fe..140523a082 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt @@ -2,35 +2,20 @@ package flank.exection.parallel.internal import flank.exection.parallel.Parallel import flank.exection.parallel.ParallelState -import flank.exection.parallel.Tasks +/** + * Lazy accessor to the state value for a give type. + */ @Suppress("UNCHECKED_CAST") -internal fun Parallel.Store.lazy( +internal fun Parallel.Context.lazy( type: Parallel.Type ): Lazy = lazy { state[type] as T } -internal interface CreateFunction { - val store: () -> S - operator fun invoke(create: suspend S.() -> T): suspend ParallelState.() -> T = { - store().also { it.state = this }.run { create() } - } +/** + * Abstract factory for creating task function. + */ +internal interface CreateTaskFunction { + val context: () -> X + operator fun invoke(body: suspend X.() -> T): suspend ParallelState.() -> T = + { context().also { it.state = this }.run { body() } } } - -tailrec fun Tasks.subgraph( - current: Tasks, - acc: Tasks = - if (current.isEmpty()) this - else emptySet(), -): Tasks = when { - current.isEmpty() -> acc - else -> subgraph( - current = current.flatMap { it.signature.args } - .mapNotNull(this::findByReturn) - .toSet(), - acc = acc + current - ) -} - -fun Tasks.findByReturn( - type: Parallel.Type<*> -) = find { it.signature.returns == type } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt new file mode 100644 index 0000000000..836b39b38f --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt @@ -0,0 +1,24 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.MissingDependencies +import flank.exection.parallel.Parallel +import flank.exection.parallel.ParallelState +import flank.exection.parallel.Tasks + +/** + * Validate the given [Tasks] and [ParallelState] for finding missing dependencies or broken paths. + * + * @param initial The initial arguments for tasks execution. + * @return Valid [Tasks] if graph has no broken paths or missing dependencies. + * @throws [Parallel.ValidationError] if find broken paths between tasks. + */ +internal fun Tasks.validate(initial: ParallelState = emptyMap()): Tasks = apply { + val missing = findMissingDependencies(initial.keys) + if (missing.isNotEmpty()) throw Parallel.ValidationError(missing) +} + +private fun Tasks.findMissingDependencies(initial: Set>): MissingDependencies { + val all = map { task -> task.signature.returns }.toSet() + initial + return associate { task -> task.signature.run { returns to args - all } } + .filterValues { missing -> missing.isNotEmpty() } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt new file mode 100644 index 0000000000..ee37f2273d --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt @@ -0,0 +1,133 @@ +package flank.exection.parallel + +import flank.exection.parallel.Example.Args +import flank.exection.parallel.Example.Hello +import flank.exection.parallel.Example.Summary +import flank.exection.parallel.Parallel.Type +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.asFlow +import kotlinx.coroutines.flow.last +import kotlinx.coroutines.flow.onEach +import kotlinx.coroutines.flow.toList +import kotlinx.coroutines.runBlocking + +// ======================= Public API ======================= + +private object Example { + + /** + * Context for [Example]. + * Wrapper for state values with exposed static accessors. + */ + class Context : Parallel.Context() { + val args by Args() + val a by A() + val b by B() + val c by C() + val hello by Hello() + } + + /** + * Arguments for [Example]. + */ + data class Args( + val wait: Long, + val a: Int, + val b: Int, + val c: Int + ) { + companion object : Type + } + + object A : Type> + object B : Type> + object C : Type> + object Summary : Type> + object Hello : Type { + object Failing : Type + } + + /** + * Factory method for creating step functions in scope of [Context]. + */ + val func = Parallel.Task.Body(::Context) + + /** + * List of tasks available for [Example] + */ + val execute by lazy { + setOf( + hello, + helloFail, + produceA, + produceB, + produceC, + groupAndCount, + ) + } + + // ======================= Internal Tasks ======================= + + private val hello = Hello using { + delay(100) + "hello" + } + + private val helloFail = Hello.Failing from setOf(Hello) using func { + throw IllegalStateException(hello) + } + + private val produceA = A using func { + (0..args.a).asFlow().onEach { + out("A" to it) + delay((0..args.wait).random()) + }.toList() + } + + private val produceB = B using func { + (0..args.b).asFlow().onEach { + out("B" to it) + delay((0..args.wait).random()) + }.toList() + } + + private val produceC = C using func { + (0..args.c).asFlow().onEach { + out("C" to it) + delay((0..args.wait).random()) + }.toList() + } + + private val groupAndCount = Summary from setOf(A, B, C) using func { + delay(args.wait) + (a + b + c) + .groupBy { it } + .mapValues { (_, values) -> values.count() } + } +} + +// ======================= Example Run ======================= + +fun main() { + runBlocking { + Example.execute( + Summary, + Hello, + Hello.Failing, + )( + Args to Args( + wait = 50, + a = 5, + b = 8, + c = 13, + ), + Parallel.Logger to { log: Any -> + println(log) + } + ).last().let { state -> + println() + state.forEach(::println) + println() + } + } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt index cafb53adaf..8d5b21572a 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt @@ -1,30 +1,68 @@ package flank.exection.parallel -import flank.exection.parallel.example.ExampleExecution -import flank.exection.parallel.example.invoke +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.last +import kotlinx.coroutines.runBlocking +import org.junit.Assert import org.junit.Test -import java.text.SimpleDateFormat -class ParallelTest { +object A : Parallel.Type +object B : Parallel.Type +object C : Parallel.Type +object WaitTime : Parallel.Type +object Sum : Parallel.Type +object Delayed : Parallel.Type +object Failed : Parallel.Type +object NotRunning : Parallel.Type + +private class Context : Parallel.Context() { + val wait by WaitTime() + val a by A() + val b by B() + val c by C() + val sum by Sum() +} + +private val async = Parallel.Task.Body(::Context) + +private val execute = setOf( + + A using { 1 }, + + B using { 2 }, + + C using { 3 }, + + Delayed using async { delay(wait) }, + + Failed from setOf(Delayed, Sum) using async { throw Exception() }, + + Sum from setOf(A, B, C) using async { a + b + c }, + + NotRunning from setOf(Failed) using { } + +) + +private val tasks = execute.associateBy { it.signature.returns } + +private val initial: ParallelState = mapOf(WaitTime to 300) + +class ExecutionKtTest { + + @Test + fun test1() { + val result = runBlocking { execute(initial).last() } + Assert.assertEquals(1, result[A]) + Assert.assertEquals(2, result[B]) + Assert.assertEquals(3, result[C]) + Assert.assertEquals(6, result[Sum]) + Assert.assertEquals(Unit, result[Delayed]) + Assert.assertTrue(result[Failed] is Exception) + Assert.assertEquals(execute.last(), result[NotRunning]) + } @Test - fun execute() { - val result = object : ExampleExecution.Context { - override val args = ExampleExecution.Args( - wait = 50, - a = 5, - b = 8, - c = 13, - ) - override val out: Output = { - when (this) { - is Parallel.Event -> println("${format.format(timestamp)} - ${type::class.simpleName}: $data") - else -> Unit - } - } - private val format = SimpleDateFormat("yyyy.MM.dd HH:mm:ss.SSS") - }.invoke() - - result.forEach { println(it) } + fun test2() { + runBlocking { (tasks - A).values.toSet()(Sum)(initial).last() } } } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt deleted file mode 100644 index 930aad19a1..0000000000 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/example/ExampleExecution.kt +++ /dev/null @@ -1,111 +0,0 @@ -package flank.exection.parallel.example - -import flank.exection.parallel.Parallel -import flank.exection.parallel.Parallel.Type -import flank.exection.parallel.example.ExampleExecution.A -import flank.exection.parallel.example.ExampleExecution.B -import flank.exection.parallel.example.ExampleExecution.C -import flank.exection.parallel.example.ExampleExecution.Context -import flank.exection.parallel.example.ExampleExecution.Failing -import flank.exection.parallel.example.ExampleExecution.Summary -import flank.exection.parallel.example.ExampleExecution.func -import flank.exection.parallel.execute -import flank.exection.parallel.from -import flank.exection.parallel.using -import kotlinx.coroutines.delay -import kotlinx.coroutines.flow.asFlow -import kotlinx.coroutines.flow.onEach -import kotlinx.coroutines.flow.toList -import kotlinx.coroutines.runBlocking - -// ======================= Public API ======================= - -object ExampleExecution { - - /** - * Context for [ExampleExecution] - */ - interface Context : Parallel.Context { - val args: Args - } - - /** - * Arguments for [ExampleExecution] - */ - data class Args( - val wait: Long, - val a: Int, - val b: Int, - val c: Int - ) - - // Type definitions for [ExampleExecution] - object A : Type> - object B : Type> - object C : Type> - object Failing : Type - object Summary : Type> - - /** - * Wrapper class for state values with exposed static accessors. - */ - class Store : Parallel.Store() { - val a by A() - val b by B() - val c by C() - } - - /** - * Factory method for creating step functions in scope of [Store]. - */ - internal val func = Parallel.Function(::Store) - - /** - * List of tasks available for [ExampleExecution] - */ - internal val tasks = setOf( - errorAfterA, - produceA, - produceB, - produceC, - groupAndCount, - ) -} - -fun Context.invoke() = runBlocking { execute(ExampleExecution.tasks, setOf(groupAndCount)) } - -// ======================= Internal Tasks ======================= - -internal val errorAfterA = Failing from setOf(A) using fun Context.() = func { - throw IllegalStateException(args.toString()) -} - -internal val produceA = A using fun Context.() = func { - (0..args.a).asFlow().onEach { - out("A" to it) - delay((0..args.wait).random()) - }.toList() -} - -internal val produceB = B using fun Context.() = func { - (0..args.b).asFlow().onEach { - out("B" to it) - delay((0..args.wait).random()) - }.toList() -} - -internal val produceC = C using fun Context.() = func { - (0..args.c).asFlow().onEach { - out("C" to it) - delay((0..args.wait).random()) - }.toList() -} - -internal val groupAndCount = Summary from setOf(A, B, C) using fun Context.() = func { - delay(args.wait) - (a + b + c) - .groupBy { it } - .mapValues { (_, values) -> values.count() } -} - - diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt new file mode 100644 index 0000000000..fe0547df58 --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt @@ -0,0 +1,44 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Parallel.Type +import flank.exection.parallel.from +import flank.exection.parallel.using +import org.junit.Test + +class ReduceKtTest { + + private object A : Type + private object B : Type + private object C : Type + private object D : Type + private object E : Type + private object F : Type + + private val execute = setOf( + A using { }, + B using { }, + C from setOf(B) using { }, + D from setOf(A, C) using { }, + E using { }, + F from setOf(E) using { }, + ) + + @Test + fun test() { + val omitted = setOf(E, F) + val expected = execute.filter { it.signature.returns !in omitted } + + val actual = execute.reduce(setOf(D)) + + assert(expected.containsAll(actual)) + assert(expected.size == actual.size) + } + + @Test + fun test2() { + + val actual = execute.filter { it.signature.returns != C }.toSet().reduce(setOf(D)) + + actual.forEach(::println) + } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/UtilsKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/UtilsKtTest.kt deleted file mode 100644 index 575428f205..0000000000 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/UtilsKtTest.kt +++ /dev/null @@ -1,23 +0,0 @@ -package flank.exection.parallel.internal - -import flank.exection.parallel.Tasks -import flank.exection.parallel.example.ExampleExecution -import flank.exection.parallel.example.errorAfterA -import flank.exection.parallel.example.produceA -import org.junit.Assert -import org.junit.Test - -class UtilsKtTest { - - @Test - fun find() { - val expected: Tasks = setOf( - errorAfterA, - produceA, - ) - val actual = ExampleExecution.tasks.subgraph(setOf(errorAfterA)) - - println(actual.map { it.signature.returns.javaClass.simpleName }) - Assert.assertEquals(expected, actual) - } -} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt new file mode 100644 index 0000000000..d9bd174765 --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt @@ -0,0 +1,46 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Parallel +import flank.exection.parallel.from +import flank.exection.parallel.using +import org.junit.Assert.* +import org.junit.Test + +class ValidateKtTest { + + private object A : Parallel.Type + private object B : Parallel.Type + private object C : Parallel.Type + private object D : Parallel.Type + + private val execute = setOf( + A using { }, + B from setOf(A) using { }, + C from setOf(B) using { }, + D from setOf(A, C) using { }, + ) + + /** + * Return empty map when there are no missing dependencies. + */ + @Test + fun test() { + execute.validate() + } + + /** + * Return missing map of dependencies dependencies + */ + @Test + fun test2() { + val expected = mapOf( + B to setOf(A), + D to setOf(A) + ) + try { + execute.drop(1).toSet().validate() + } catch (e: Parallel.ValidationError) { + assertEquals(expected, e.data) + } + } +} From e18d08965f683006bc3843c5f8be9d18b7166f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 7 Jun 2021 16:45:50 +0200 Subject: [PATCH 04/29] Add unit tests --- .../kotlin/flank/exection/parallel/Ext.kt | 23 +++ .../flank/exection/parallel/Parallel.kt | 22 +- .../exection/parallel/internal/Execution.kt | 15 +- .../flank/exection/parallel/ParallelTest.kt | 189 +++++++++++++----- 4 files changed, 198 insertions(+), 51 deletions(-) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt index 50b4ae1740..df37fd58a5 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt @@ -1,5 +1,7 @@ package flank.exection.parallel +import kotlinx.coroutines.Job + /** * Extension for executing tasks on a given state. */ @@ -7,9 +9,30 @@ operator fun Tasks.invoke( vararg state: Pair, Any> ) = invoke(state.toMap()) +/** + * Extension for executing tasks on empty state. + */ +operator fun Tasks.invoke() = + invoke(emptyMap()) + /** * Extension for reducing tasks to returned with dependencies. */ operator fun Tasks.invoke( vararg returns: Parallel.Type<*> ) = invoke(returns.toSet()) + + +data class DeadlockError( + val state: ParallelState, + val jobs: Map, Job>, + val remaining: Map>, List>>, +) : Error() { + override fun toString(): String = listOf( + "Parallel execution dump:", + state, + jobs, + remaining, + "", super.toString(), + ).joinToString("\n") +} diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt index 1ca890ab70..c268996a6d 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -6,19 +6,30 @@ import flank.exection.parallel.internal.invoke import flank.exection.parallel.internal.lazy import flank.exection.parallel.internal.reduce import flank.exection.parallel.internal.validate +import kotlinx.coroutines.flow.Flow import java.lang.System.currentTimeMillis /** * Reduce given [Tasks] by [returns] types to remove unneeded tasks from the graph. * The returned graph will contain only tasks that are returning selected types, their dependencies and derived dependencies. + * + * @return Reduced [Tasks] */ -operator fun Tasks.invoke(returns: Set>): Tasks = reduce(returns) +operator fun Tasks.invoke( + returns: Set> +): Tasks = + reduce(returns) /** * Execute tasks in parallel with a given args. * Before executing the [validate] is performed on a [Tasks] for a given [args] to detect missing dependencies. + * + * @return [Flow] of [ParallelState] changes. */ -infix operator fun Tasks.invoke(args: ParallelState) = Execution(validate(args), args).invoke() +infix operator fun Tasks.invoke( + args: ParallelState +): Flow = + Execution(validate(args), args).invoke() /** * The namespace for parallel execution primitives. @@ -56,7 +67,7 @@ object Parallel { */ data class Task( val signature: Signature, - val execute: suspend ParallelState.() -> R + val execute: ExecuteTask ) { /** * The task signature that contains arguments types and returned type. @@ -88,6 +99,11 @@ object Parallel { ) : Exception("Cannot resolve dependencies $data") } +/** + * Signature for [Parallel.Task] function. + */ +internal typealias ExecuteTask = suspend ParallelState.() -> R + /** * Common signature for structural log output. */ diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt index 61238939d2..cc10bc6624 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt @@ -1,5 +1,6 @@ package flank.exection.parallel.internal +import flank.exection.parallel.DeadlockError import flank.exection.parallel.Output import flank.exection.parallel.Parallel.Event import flank.exection.parallel.Parallel.Logger @@ -14,6 +15,7 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.consumeAsFlow +import kotlinx.coroutines.flow.drop import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.scan @@ -55,6 +57,9 @@ internal operator fun Execution.invoke(): Flow = // Cancel the coroutine scope for tasks. .onCompletion { scope.cancel() } + // Drop first element which is always the initial state + .drop(1) + /** * Internal context of the tasks execution. */ @@ -111,10 +116,11 @@ private fun Execution.isNotClosed(): Boolean = !channel.isClosedForSend /** * Cancel running jobs, update state for cancellation values and close the execution. */ -private fun Execution.abortBy(type: Type<*>, cause: Throwable) { +private suspend fun Execution.abortBy(type: Type<*>, cause: Throwable) { // Abort running jobs. scope.cancel("aborted by ${type.javaClass.name}", cause) + scope.coroutineContext[Job]?.join() // Add remaining tasks to state. remaining.toMap().values.flatten().forEach { task -> @@ -149,7 +155,12 @@ private fun Execution.filterTasksFor(state: ParallelState): Map>, Li .apply { remaining -= keys } // Check if execution is not detached. Typically will fail if broken graph was not validated before execution and passed here. - .apply { check(isNotEmpty() || jobs.any { (_, job) -> job.isActive }) { "Deadlock!" } } + .apply { + isNotEmpty() + || jobs.any { (_, job) -> job.isActive } + || channel.isEmpty.not() + || throw DeadlockError(state, jobs, remaining) + } /** * Execute given tasks as coroutine job. diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt index 8d5b21572a..c5759ed5a9 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt @@ -1,68 +1,165 @@ package flank.exection.parallel +import kotlinx.coroutines.CancellationException import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.last +import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking -import org.junit.Assert +import kotlinx.coroutines.withTimeout +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue import org.junit.Test +import java.util.concurrent.atomic.AtomicInteger -object A : Parallel.Type -object B : Parallel.Type -object C : Parallel.Type -object WaitTime : Parallel.Type -object Sum : Parallel.Type -object Delayed : Parallel.Type -object Failed : Parallel.Type -object NotRunning : Parallel.Type +// ====================== COMMON TYPES ====================== -private class Context : Parallel.Context() { - val wait by WaitTime() - val a by A() - val b by B() - val c by C() - val sum by Sum() -} - -private val async = Parallel.Task.Body(::Context) - -private val execute = setOf( - - A using { 1 }, - - B using { 2 }, +object IntType : Parallel.Type +object A : Parallel.Type +object B : Parallel.Type +object C : Parallel.Type +object D : Parallel.Type +object E : Parallel.Type - C using { 3 }, +// ====================== TESTS ====================== - Delayed using async { delay(wait) }, +class ParallelTest { - Failed from setOf(Delayed, Sum) using async { throw Exception() }, - - Sum from setOf(A, B, C) using async { a + b + c }, - - NotRunning from setOf(Failed) using { } + /** + * Executing [Tasks] will return empty flow of [ParallelState]. + */ + @Test + fun `run empty execution`() { + val execute: Tasks = emptySet() + val expected = listOf() + val actual = runBlocking { execute().toList() } + assertEquals(expected, actual) + } -) + /** + * Executing a [Parallel.Task] is producing initial state with accumulated value. + */ + @Test + fun `run single task`() { + val execute = setOf(IntType using { 1 }) + val expected = listOf(mapOf(IntType to 1)) + val actual = runBlocking { execute().toList() } + assertEquals(expected, actual) + } -private val tasks = execute.associateBy { it.signature.returns } + /** + * Valid graph of task of task is run in optimized order. + */ + @Test + fun `run many tasks`() { + val counter = AtomicInteger() + val waitTime = (0..50L) + val nextValue: ExecuteTask = { + delay(waitTime.random()) + counter.accumulateAndGet(1) { l, r -> l + r } + } + val execute = setOf( + A from setOf(B, C) using nextValue, + B from setOf(D, E) using nextValue, + C using nextValue, + D using nextValue, + E using nextValue, + ) + val expectedOrder = mapOf( + E to setOf(1, 2, 3), + D to setOf(1, 2, 3), + C to setOf(1, 2, 3, 4), + B to setOf(3, 4), + A to setOf(5), + ) + val actual = runBlocking { execute().last() } + + assert( + actual.all { (key, value) -> value in expectedOrder[key]!! } + ) { + expectedOrder.map { (key, expected) -> key to (actual[key] to expected) }.joinToString("\n", prefix = "\n") + } + } -private val initial: ParallelState = mapOf(WaitTime to 300) + /** + * Several random executions for detecting concurrent issues. + */ + @Test + fun `run randomized stress test`() = runBlocking { + data class Type(val id: Int) : Parallel.Type + repeat(100) { + withTimeout(1000) { + // given + val types = (0..100).map { Type(it) } + val used = mutableSetOf>() + val execute: Tasks = types.map { returns -> + used += returns + val args = (types - used) + .shuffled() + .run { take((0..size).random()) } + .toSet() + returns from args using {} + }.toSet() + // when + runBlocking { execute().collect() } + // then expect no errors + } + } + } -class ExecutionKtTest { + /** + * For failing task, the execution is accumulating [Throwable] instead of value with expected type. + */ + @Test + fun `run failing task`() { + val exception = Exception() + val execute = setOf(IntType using { throw exception }) + val expected = listOf(mapOf(IntType to exception)) + val actual = runBlocking { execute().toList() } + assertEquals(expected, actual) + } + /** + * Single failing task is aborting all running and remaining tasks. + */ @Test - fun test1() { - val result = runBlocking { execute(initial).last() } - Assert.assertEquals(1, result[A]) - Assert.assertEquals(2, result[B]) - Assert.assertEquals(3, result[C]) - Assert.assertEquals(6, result[Sum]) - Assert.assertEquals(Unit, result[Delayed]) - Assert.assertTrue(result[Failed] is Exception) - Assert.assertEquals(execute.last(), result[NotRunning]) + fun `abort execution`() { + val execute = setOf( + A using { delay(50); throw Exception() }, + B using { delay(100) }, + C from setOf(B) using {} + ) + val actual = runBlocking { execute().last() } + assertTrue(actual[B].toString(), actual[B] is CancellationException) + assertTrue(actual[C].toString(), actual[C] is Parallel.Task<*>) } + /** + * Task can access only initial and specified properties + */ @Test - fun test2() { - runBlocking { (tasks - A).values.toSet()(Sum)(initial).last() } + fun `hermetic context`() { + class Context : Parallel.Context() { + val initial by D() + val b by B() + val c by C() + } + + val func = Parallel.Task.Body(::Context) + val execute = setOf( + A from setOf(B) using func { + initial + b + c // accessing this property will throw exception + }, + B from setOf(C) using { 1 }, + C using { 2 } + ) + val args: ParallelState = mapOf( + D to 3 + ) + val actual = runBlocking { execute(args).last() } + + assert(actual[A] is NullPointerException) } } From 8afc135d1f85cf1c42205439320cf14aaa3f1abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 9 Jun 2021 12:38:20 +0200 Subject: [PATCH 05/29] Add unfinished doc Add validation for initial state --- docs/hld/parallel-execution-api.puml | 79 ++++++++++++++++ docs/hld/task-graph.puml | 89 +++++++++++++++++++ tool/execution/parallel/README.md | 64 +++++++++++++ .../kotlin/flank/exection/parallel/Ext.kt | 39 +++----- .../kotlin/flank/exection/parallel/Factory.kt | 17 ++++ .../flank/exection/parallel/Parallel.kt | 58 ++++++++---- .../{Utils.kt => CreateTaskFunction.kt} | 13 +-- .../exection/parallel/internal/Execution.kt | 38 +++++--- .../exection/parallel/internal/Prepare.kt | 11 +++ .../exection/parallel/internal/Property.kt | 34 +++++++ .../exection/parallel/internal/Reduce.kt | 6 +- .../exection/parallel/internal/Validate.kt | 18 +++- .../kotlin/flank/exection/parallel/Example.kt | 38 +++++--- .../flank/exection/parallel/ParallelTest.kt | 22 ++++- .../parallel/internal/ReduceKtTest.kt | 4 +- .../parallel/internal/ValidateKtTest.kt | 2 +- 16 files changed, 443 insertions(+), 89 deletions(-) create mode 100644 docs/hld/parallel-execution-api.puml create mode 100644 docs/hld/task-graph.puml create mode 100644 tool/execution/parallel/README.md rename tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/{Utils.kt => CreateTaskFunction.kt} (54%) create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Property.kt diff --git a/docs/hld/parallel-execution-api.puml b/docs/hld/parallel-execution-api.puml new file mode 100644 index 0000000000..a371776875 --- /dev/null +++ b/docs/hld/parallel-execution-api.puml @@ -0,0 +1,79 @@ +@startuml +package "Parallel" { + +class Context +class Task +class "Task.Signature" as Task_Signature +class "Task.Body" as Task_Body +interface ExecuteTask +interface Type +interface ParallelState +interface Output + +} + +Output "0..1" ..o "1" Context +ParallelState "1" --o "1" Context +Type <|-- Context + +Type "1" --* "1" Task_Signature +Type "*" --o "1" Task_Signature +Task_Signature "1" --* "*" Task + +Task_Body "1" ..> "*" ExecuteTask +ExecuteTask "1" --* "1" Task + +@enduml + +@startuml +'left to right direction +package "parallel" { + +object "Task.Body.invoke" as Task_Body + +class Task +class "Task.Signature" as Task_Signature +class ParallelStateFlow +interface Type +interface ExecuteTask +interface ParallelState + +'package "Factory" { +object "using" as usingType +object "using" as usingSignature +object from +usingSignature -right- usingType +usingType -right- from +'} +object "invoke" as invokeExecution { +execute tasks +} +object "invoke" as reduce { +reduce tasks +} + +Task_Body ..> ExecuteTask + +ExecuteTask <-- usingSignature +usingSignature --# Task_Signature +usingSignature ..> Task +usingSignature --* usingType + +Type #-- usingType +ExecuteTask <-- usingType +usingType ..> Task + +Type <-- from +Type #-- from +from ..> Task_Signature +from -* usingType + +reduce <- Task +reduce .> Task + +Task --> invokeExecution +invokeExecution <- ParallelState +invokeExecution ..> ParallelStateFlow + +} +@enduml diff --git a/docs/hld/task-graph.puml b/docs/hld/task-graph.puml new file mode 100644 index 0000000000..13c69f9a22 --- /dev/null +++ b/docs/hld/task-graph.puml @@ -0,0 +1,89 @@ +@startuml +package "Valid graph" { +map H1 #lightgreen { +} +map I1 #lightgreen { +} +map F1 #lightgreen { +I1 *--> I1 +} +map E1 #lightgreen { +I1 *--> I1 +} +map D1 #lightgreen { +I1 *--> I1 +} +map C1 #lightgreen { +D1 *--> D1 +} +map B1 #lightgreen { +C1 *-> C1 +E1 *--> E1 +H1*---> H1 +} +map A1 #lightgreen { +B1 *--> B1 +F1 *--> F1 +C1 *--> C1 +I1 *--> I1 +} +} +package "Selected graph" { +map H3 #lightgreen { +} +map I3 #lightgreen { +} +map F3 #lightgrey { +I3 *--> I3 +} +map E3 #lightgreen { +I3 *--> I3 +} +map D3 #lightgreen { +I3 *--> I3 +} +map C3 #lightgreen { +D3 *--> D3 +} +map B3 #lightblue { +C3 *-> C3 +E3 *--> E3 +H3*---> H3 +} +map A3 #lightgrey { +B3 *--> B3 +F3 *--> F3 +C3 *--> C3 +I3 *--> I3 +} +} +package "Broken graph" { +map H2 #lightgrey { +} +map I2 #black { +} +map F2 #red { +I2 *--> I2 +} +map E2 #red { +I2 *--> I2 +} +map D2 #red { +I2 *--> I2 +} +map C2 #yellow { +D2 *--> D2 +} +map B2 #yellow { +C2 *-> C2 +E2 *--> E2 +H2*---> H2 +} +map A2 #yellow { +B2 *--> B2 +F2 *--> F2 +C2 *--> C2 +I2 *--> I2 +} +} +@enduml diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md new file mode 100644 index 0000000000..36562a1d54 --- /dev/null +++ b/tool/execution/parallel/README.md @@ -0,0 +1,64 @@ +# Parallel Execution + +Library for task parallelization and asynchronous execution. + +### References + +* Module type - [tool](../../docs/architecture.md#tool) +* Dependency type - [static](../../docs/architecture.md#static_dependencies) +* Public API + * Core - [Parallel.kt](./src/main/kotlin/flank/exection/parallel/Parallel.kt) + * Task factories DSL - [Factory.kt](./src/main/kotlin/flank/exection/parallel/Factory.kt) + * Extensions - [Ext.kt](./src/main/kotlin/flank/exection/parallel/Ext.kt) + +# Design + +1. Imagine a complicated long-running `execution` that needs to perform several operations (`tasks`) in correct order to collect a required `data` and produce `side effects`. +2. Any `execution` like that, can be modeled as the set of unrelated data `types` and suspendable functions (`tasks`). + +## Execution + +* will run tasks in optimized order creating synchronization points where needed and running all other stuff in parallel. +* is collecting result from each task, and accumulating it to state. +* is returning the flow of realtime state changes. +* can detach and fail if set of `tasks` are creating broken graph with `missing dependencies`. +* Each task in execution scope must have unique return type. This is crucial for producing correct graph. + +The following diagram is showing parallel execution algorithm in details: + +![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution.puml) + +## Type + +* is atomic and unique in the execution scope. +* is describing the data type for value that can be: + * passed to execution as arguments for initial state. + * passed to task from state as argument. + * returned from task. + * returned from execution. + +## Task + +* `arguments` for task are coming from `initial state` (`arguments`) or results from other `tasks`. +* can be grouped into set to create the `execution` graph. +* graph can have broken paths, when any task have `missing dependencies`. + +### Signature + +* is a set of `argument` `types` related with one `return` `type` just like in normal functions. + +### Function + +* is getting access to arguments through reduced copy of state. +* is returning object described by the return type. + +### Missing dependencies + +* missing dependencies can be detected before execution from the set of `tasks` and `initial state`. + +## State + +* is a map of types with associated values that are coming from initial arguments or task results. +* is storing the result from each executed task. +* is used for providing arguments for tasks. +* is emitted in flow after each change. diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt index df37fd58a5..e5f17ee202 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt @@ -1,38 +1,25 @@ package flank.exection.parallel -import kotlinx.coroutines.Job +import kotlinx.coroutines.flow.Flow +// ======================= Reduce ======================= /** - * Extension for executing tasks on a given state. + * Shortcut for tasks reducing. */ operator fun Tasks.invoke( - vararg state: Pair, Any> -) = invoke(state.toMap()) + vararg returns: Parallel.Type<*> +): Tasks = invoke(returns.toSet()) +// ======================= Execute ======================= /** - * Extension for executing tasks on empty state. + * Shortcut for executing tasks on a given state. */ -operator fun Tasks.invoke() = - invoke(emptyMap()) +operator fun Tasks.invoke( + vararg state: Pair, Any> +): Flow = invoke(state.toMap()) /** - * Extension for reducing tasks to returned with dependencies. + * Shortcut for executing tasks on empty state. */ -operator fun Tasks.invoke( - vararg returns: Parallel.Type<*> -) = invoke(returns.toSet()) - - -data class DeadlockError( - val state: ParallelState, - val jobs: Map, Job>, - val remaining: Map>, List>>, -) : Error() { - override fun toString(): String = listOf( - "Parallel execution dump:", - state, - jobs, - remaining, - "", super.toString(), - ).joinToString("\n") -} +operator fun Tasks.invoke(): Flow = + invoke(emptyMap()) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt index dc59332dc8..91b83827cd 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt @@ -1,5 +1,9 @@ package flank.exection.parallel +import flank.exection.parallel.internal.EagerProperties + +// ======================= Signature ======================= + /** * Factory function for creating [Parallel.Task.Signature] from expected type and argument types. */ @@ -8,6 +12,8 @@ infix fun Parallel.Type.from( ): Parallel.Task.Signature = Parallel.Task.Signature(this, args) +// ======================= Task ======================= + /** * Factory function for creating [Parallel.Task] from signature and execute function. */ @@ -23,3 +29,14 @@ infix fun Parallel.Type.using( execute: suspend ParallelState.() -> R ): Parallel.Task = from(emptySet()).using(execute) + +// ======================= Validator ======================= + +/** + * Factory function for creating special task that can validate arguments before execution. + * Typically the [Parallel.Context] with added [EagerProperties] is used to validate initial state. + */ +internal fun validator( + context: (() -> C) +): Parallel.Task = + context() using { context().also { it.state = this }.run { eager() } } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt index c268996a6d..011f49ed67 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -1,28 +1,33 @@ package flank.exection.parallel import flank.exection.parallel.internal.CreateTaskFunction +import flank.exection.parallel.internal.EagerProperties import flank.exection.parallel.internal.Execution +import flank.exection.parallel.internal.initialValidators import flank.exection.parallel.internal.invoke -import flank.exection.parallel.internal.lazy -import flank.exection.parallel.internal.reduce +import flank.exection.parallel.internal.lazyProperty +import flank.exection.parallel.internal.reduceTo import flank.exection.parallel.internal.validate import kotlinx.coroutines.flow.Flow import java.lang.System.currentTimeMillis +// ======================= Core functions ======================= + /** - * Reduce given [Tasks] by [returns] types to remove unneeded tasks from the graph. - * The returned graph will contain only tasks that are returning selected types, their dependencies and derived dependencies. + * Reduce given [Tasks] by [select] types to remove unneeded tasks from the graph. + * The returned graph will only tasks that are returning selected types, their dependencies and derived dependencies. + * Additionally this is keeping also the validators for initial state. * * @return Reduced [Tasks] */ operator fun Tasks.invoke( - returns: Set> + select: Set> ): Tasks = - reduce(returns) + reduceTo(select + initialValidators) /** * Execute tasks in parallel with a given args. - * Before executing the [validate] is performed on a [Tasks] for a given [args] to detect missing dependencies. + * Before executing, the [validate] is performed on a [Tasks] for a given [args] to detect missing dependencies. * * @return [Flow] of [ParallelState] changes. */ @@ -31,6 +36,8 @@ infix operator fun Tasks.invoke( ): Flow = Execution(validate(args), args).invoke() +// ======================= Types ======================= + /** * The namespace for parallel execution primitives. */ @@ -40,21 +47,34 @@ object Parallel { * Abstraction for execution data provider which is also an context for task execution. * For initialization purpose some properties are exposed as variable. */ - open class Context { + open class Context : Type { + + /** + * Exposed reference to output, accessible by tasks. + */ + val out: Output by -Logger + /** * The state properties map. Is for initialization purpose, and shouldn't be mutated after initialization. */ internal lateinit var state: ParallelState /** - * Exposed reference to output, accessible by tasks. + * Eager property delegate provider and initializer of a state values. + */ + val eager = EagerProperties { state } + + /** + * DSL for creating eager delegate accessor to the state value for a given type. + * + * @see [EagerProperties.invoke] */ - val out: Output by Logger() + inline operator fun Type.not() = eager(this) /** - * Factory method for creating properties from data objects. + * DSL for creating lazy delegate accessor to the state value for a given type. */ - operator fun Type.invoke(): Lazy = lazy(type = this) + operator fun Type.unaryMinus() = lazyProperty(this) } /** @@ -83,7 +103,7 @@ object Parallel { class Body(override val context: () -> X) : CreateTaskFunction } - data class Event( + data class Event internal constructor( val type: Type<*>, val data: Any, val timestamp: Long = currentTimeMillis(), @@ -99,27 +119,29 @@ object Parallel { ) : Exception("Cannot resolve dependencies $data") } +// ======================= Aliases ======================= + /** * Signature for [Parallel.Task] function. */ -internal typealias ExecuteTask = suspend ParallelState.() -> R +typealias ExecuteTask = suspend ParallelState.() -> R /** * Common signature for structural log output. */ -internal typealias Output = Any.() -> Unit +typealias Output = Any.() -> Unit /** * Immutable state for parallel execution. */ -internal typealias ParallelState = Map, Any> +typealias ParallelState = Map, Any> /** * Type for group of parallel tasks. Each task must be unique in group. */ -internal typealias Tasks = Set> +typealias Tasks = Set> /** * The [Map.values] are representing missing dependencies required by the tasks to provide [Map.keys]. */ -internal typealias MissingDependencies = Map, Set>> +typealias MissingDependencies = Map, Set>> diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/CreateTaskFunction.kt similarity index 54% rename from tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt rename to tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/CreateTaskFunction.kt index 140523a082..0ac12e33e3 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Utils.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/CreateTaskFunction.kt @@ -3,19 +3,12 @@ package flank.exection.parallel.internal import flank.exection.parallel.Parallel import flank.exection.parallel.ParallelState -/** - * Lazy accessor to the state value for a give type. - */ -@Suppress("UNCHECKED_CAST") -internal fun Parallel.Context.lazy( - type: Parallel.Type -): Lazy = lazy { state[type] as T } - /** * Abstract factory for creating task function. */ internal interface CreateTaskFunction { val context: () -> X - operator fun invoke(body: suspend X.() -> T): suspend ParallelState.() -> T = - { context().also { it.state = this }.run { body() } } + operator fun invoke(body: suspend X.() -> T): suspend ParallelState.() -> T = { + context().also { it.state = this }.run { body() } + } } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt index cc10bc6624..129055c4c0 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt @@ -1,6 +1,5 @@ package flank.exection.parallel.internal -import flank.exection.parallel.DeadlockError import flank.exection.parallel.Output import flank.exection.parallel.Parallel.Event import flank.exection.parallel.Parallel.Logger @@ -149,18 +148,35 @@ private fun Execution.isFinished(state: ParallelState): Boolean = * @throws IllegalStateException If there are no tasks to run and no running jobs for a given state. */ private fun Execution.filterTasksFor(state: ParallelState): Map>, List>> = - remaining.filterKeys(state.keys::containsAll) + remaining.filterKeys(state.keys::containsAll).apply { + + // Check if execution is not detached, other words some of tasks cannot be run because of missing values in state. + // Typically will fail if broken graph was not validated before execution. + isNotEmpty() || // Found tasks to execute. + jobs.any { (_, job) -> job.isActive } || // some jobs still are running, is required to wait for values from them. + channel.isEmpty.not() || // some values are waiting in the channel queue. + throw DeadlockError(state, jobs, remaining) // Remove from queue the tasks for current iteration. - .apply { remaining -= keys } - - // Check if execution is not detached. Typically will fail if broken graph was not validated before execution and passed here. - .apply { - isNotEmpty() - || jobs.any { (_, job) -> job.isActive } - || channel.isEmpty.not() - || throw DeadlockError(state, jobs, remaining) - } + remaining -= keys + } + +/** + * Typically can occurs when execution is running on broken graph of tasks. + */ +private data class DeadlockError( + val state: ParallelState, + val jobs: Map, Job>, + val remaining: Map>, List>>, +) : Error() { + override fun toString(): String = listOf( + "Parallel execution dump:", + state, + jobs, + remaining, + "", super.toString(), + ).joinToString("\n") +} /** * Execute given tasks as coroutine job. diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt new file mode 100644 index 0000000000..dc9c3b6efc --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt @@ -0,0 +1,11 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Parallel +import flank.exection.parallel.Tasks + +/** + * Get initial state validators. + * This is necessary to perform validations of initial state before the execution. + */ +internal val Tasks.initialValidators: List + get() = mapNotNull { task -> task.signature.returns as? Parallel.Context } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Property.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Property.kt new file mode 100644 index 0000000000..615fb00bec --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Property.kt @@ -0,0 +1,34 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Parallel +import flank.exection.parallel.ParallelState + +/** + * Factory function for lazy property delegate. + */ +internal fun Parallel.Context.lazyProperty(type: Parallel.Type) = lazy { + state[type] as T +} + +/** + * Eager properties provider and initializer. + */ +class EagerProperties( + private val state: () -> ParallelState +) { + private val set = mutableSetOf>() + + /** + * Initialize eager properties, this performs also validation. + */ + operator fun invoke(): Unit = set.forEach { prop -> println(prop.value) } + + /** + * Register new parallel type. Inline modifier is necessary to perform real type check + */ + inline operator fun invoke(type: Parallel.Type): Lazy = lazy { type.value() as T }.append() + + // local helpers need to be public because of inlined invoke + fun Lazy.append(): Lazy = apply { set.plusAssign(this) } + fun Parallel.Type.value(): Any? = state()[this] +} diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt index 4cc3897d68..18d23d2213 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt @@ -3,10 +3,10 @@ package flank.exection.parallel.internal import flank.exection.parallel.Parallel import flank.exection.parallel.Tasks -internal infix fun Tasks.reduce( - select: Set> +internal infix fun Tasks.reduceTo( + selectedTypes: Set> ): Tasks = - filter { task -> task.signature.returns in select } + filter { task -> task.signature.run { returns in selectedTypes } } .toSet() .reduce(this) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt index 836b39b38f..585c6fd463 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt @@ -4,6 +4,7 @@ import flank.exection.parallel.MissingDependencies import flank.exection.parallel.Parallel import flank.exection.parallel.ParallelState import flank.exection.parallel.Tasks +import kotlinx.coroutines.runBlocking /** * Validate the given [Tasks] and [ParallelState] for finding missing dependencies or broken paths. @@ -12,11 +13,22 @@ import flank.exection.parallel.Tasks * @return Valid [Tasks] if graph has no broken paths or missing dependencies. * @throws [Parallel.ValidationError] if find broken paths between tasks. */ -internal fun Tasks.validate(initial: ParallelState = emptyMap()): Tasks = apply { - val missing = findMissingDependencies(initial.keys) - if (missing.isNotEmpty()) throw Parallel.ValidationError(missing) +internal fun Tasks.validate(initial: ParallelState = emptyMap()): Tasks = run { + val (validators, tasks) = splitTasks() + + // check if initial state is providing all required values specified in context. + runBlocking { validators.forEach { check -> check.execute(initial) } } + + // validate graph is not broken + findMissingDependencies(initial.keys).run { if (isNotEmpty()) throw Parallel.ValidationError(this) } + + tasks.toSet() } +private fun Iterable>.splitTasks() = this + .groupBy { it.signature.returns is Parallel.Context } + .run { getOrDefault(true, emptyList()) to getOrDefault(false, emptyList()) } + private fun Tasks.findMissingDependencies(initial: Set>): MissingDependencies { val all = map { task -> task.signature.returns }.toSet() + initial return associate { task -> task.signature.run { returns to args - all } } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt index ee37f2273d..435a4e166a 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt @@ -16,15 +16,18 @@ import kotlinx.coroutines.runBlocking private object Example { /** - * Context for [Example]. + * Context for [Example] scope. * Wrapper for state values with exposed static accessors. */ class Context : Parallel.Context() { - val args by Args() - val a by A() - val b by B() - val c by C() - val hello by Hello() + // Initial part of state. + // Validated before execution + val args by !Args + // Values evaluated during execution by tasks. + val a by -A + val b by -B + val c by -C + val hello by -Hello } /** @@ -36,6 +39,9 @@ private object Example { val b: Int, val c: Int ) { + /** + * Specify type for [Args] so can be added to [ParallelState] + */ companion object : Type } @@ -48,15 +54,16 @@ private object Example { } /** - * Factory method for creating step functions in scope of [Context]. + * Factory method for creating task functions with [Context]. */ val func = Parallel.Task.Body(::Context) /** - * List of tasks available for [Example] + * List of tasks in [Example] scope */ - val execute by lazy { + val execute: Tasks by lazy { setOf( + validate, hello, helloFail, produceA, @@ -68,7 +75,9 @@ private object Example { // ======================= Internal Tasks ======================= - private val hello = Hello using { + private val validate: Parallel.Task = validator(::Context) + + private val hello: Parallel.Task = Hello using { delay(100) "hello" } @@ -111,10 +120,14 @@ private object Example { fun main() { runBlocking { Example.execute( + // Expect selected tasks. Summary, Hello, - Hello.Failing, + // Uncomment to simulate failing execution. + // Hello.Failing, )( + // Specify initial state. + // Commend initial Args to simulate failure of initial validation. Args to Args( wait = 50, a = 5, @@ -123,8 +136,9 @@ fun main() { ), Parallel.Logger to { log: Any -> println(log) - } + }, ).last().let { state -> + // Print final state println() state.forEach(::println) println() diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt index c5759ed5a9..cec79b4e6d 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt @@ -10,6 +10,7 @@ import kotlinx.coroutines.withTimeout import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test +import java.lang.ClassCastException import java.util.concurrent.atomic.AtomicInteger // ====================== COMMON TYPES ====================== @@ -140,9 +141,9 @@ class ParallelTest { @Test fun `hermetic context`() { class Context : Parallel.Context() { - val initial by D() - val b by B() - val c by C() + val initial by !D + val b by -B + val c by -C } val func = Parallel.Task.Body(::Context) @@ -162,4 +163,19 @@ class ParallelTest { assert(actual[A] is NullPointerException) } + + /** + * If the validator function is specified in tasks it will validate arguments before running the execution. + * The failing validation will throw [NullPointerException] or [ClassCastException] + */ + @Test(expected = ClassCastException::class) + fun `validate arguments`() { + class Context : Parallel.Context() { + val initial by !IntType + } + + val args: ParallelState = mapOf(IntType to "asd") + val execute = setOf(validator(::Context)) + runBlocking { execute(args).last() } + } } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt index fe0547df58..ca2e39f4ce 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt @@ -28,7 +28,7 @@ class ReduceKtTest { val omitted = setOf(E, F) val expected = execute.filter { it.signature.returns !in omitted } - val actual = execute.reduce(setOf(D)) + val actual = execute.reduceTo(setOf(D)) assert(expected.containsAll(actual)) assert(expected.size == actual.size) @@ -37,7 +37,7 @@ class ReduceKtTest { @Test fun test2() { - val actual = execute.filter { it.signature.returns != C }.toSet().reduce(setOf(D)) + val actual = execute.filter { it.signature.returns != C }.toSet().reduceTo(setOf(D)) actual.forEach(::println) } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt index d9bd174765..c599612fb2 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt @@ -3,7 +3,7 @@ package flank.exection.parallel.internal import flank.exection.parallel.Parallel import flank.exection.parallel.from import flank.exection.parallel.using -import org.junit.Assert.* +import org.junit.Assert.assertEquals import org.junit.Test class ValidateKtTest { From 181841eb42fd447ef0511af14c4e936f8f6bd332 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 9 Jun 2021 16:53:39 +0200 Subject: [PATCH 06/29] Make context dsl protected Split & Update diagram --- .../hld/parallel-execution-api-functions.puml | 58 ++++++++++++++ .../parallel-execution-api-structures.puml | 42 ++++++++++ docs/hld/parallel-execution-api.puml | 79 ------------------- docs/hld/parallel-execution.puml | 10 +-- tool/execution/parallel/README.md | 4 + .../kotlin/flank/exection/parallel/Factory.kt | 2 +- .../flank/exection/parallel/Parallel.kt | 11 ++- 7 files changed, 118 insertions(+), 88 deletions(-) create mode 100644 docs/hld/parallel-execution-api-functions.puml create mode 100644 docs/hld/parallel-execution-api-structures.puml delete mode 100644 docs/hld/parallel-execution-api.puml diff --git a/docs/hld/parallel-execution-api-functions.puml b/docs/hld/parallel-execution-api-functions.puml new file mode 100644 index 0000000000..1f2f08d87b --- /dev/null +++ b/docs/hld/parallel-execution-api-functions.puml @@ -0,0 +1,58 @@ +@startuml +'left to right direction +package "parallel" { + +object "Task.Body.invoke()" as Task_Body { +creates +} + +class Task +class "Task.Signature" as Task_Signature +interface Type +interface "Type" as Type2 +interface ExecuteTask +interface ParallelState + +object "using" as usingType { +creates +} +object "using" as usingSignature { +creates +} +object from { +creates +} + +object "invoke()" as invokeExecution { +executes +} +object "invoke()" as reduce { +reduces +} + +Task_Body "1" ..> ExecuteTask + +ExecuteTask <--o "1" usingSignature +usingSignature "1" #--> Task_Signature +usingSignature "1" ..> Task +usingSignature -right-* usingType + +Type <--# "1" usingType +ExecuteTask <--o "1" usingType +usingType "1" ..> Task +usingType *-left- from + +Type <--# "1" from +Type <--o "*" from +from "1" ..> Task_Signature + +Type2 <--o "*" reduce +Task <. "*" reduce +Task <-# "*" reduce + +Task <--# "*" invokeExecution +invokeExecution "1" o--> ParallelState +invokeExecution "*" ..> ParallelState + +} +@enduml diff --git a/docs/hld/parallel-execution-api-structures.puml b/docs/hld/parallel-execution-api-structures.puml new file mode 100644 index 0000000000..28bf68be8b --- /dev/null +++ b/docs/hld/parallel-execution-api-structures.puml @@ -0,0 +1,42 @@ +@startuml + + +package "parallel" { + +class "Parallel.Context" as Context { +out: Output? +} +class "Parallel.Task" as Task { +signature: Signature +execute: ExecuteTask +} +class "Parallel.Task.Signature" as Task_Signature{ +returns: Type +args: Set> +} +class "Parallel.Task.Body" as Task_Body { += (X.() -> R) -> ParallelState.() -> R +} +interface ExecuteTask << (T, orchid) >> { += ParallelState.() -> R +} +interface "Parallel.Type" as Type +interface ParallelState << (T, orchid) >> { += Map, Any> +} +interface Output << (T, orchid) >> { += Any.() -> Unit +} + +Output "0..1" ..o "1" Context +ParallelState "1" --o "1" Context +Type <|-- Context + +Type "1" --* "1" Task_Signature +Type "*" --o "1" Task_Signature +Task_Signature "1" --* "*" Task + +Task_Body "1" ..> "*" ExecuteTask +ExecuteTask "1" --* "1" Task + +@enduml diff --git a/docs/hld/parallel-execution-api.puml b/docs/hld/parallel-execution-api.puml deleted file mode 100644 index a371776875..0000000000 --- a/docs/hld/parallel-execution-api.puml +++ /dev/null @@ -1,79 +0,0 @@ -@startuml -package "Parallel" { - -class Context -class Task -class "Task.Signature" as Task_Signature -class "Task.Body" as Task_Body -interface ExecuteTask -interface Type -interface ParallelState -interface Output - -} - -Output "0..1" ..o "1" Context -ParallelState "1" --o "1" Context -Type <|-- Context - -Type "1" --* "1" Task_Signature -Type "*" --o "1" Task_Signature -Task_Signature "1" --* "*" Task - -Task_Body "1" ..> "*" ExecuteTask -ExecuteTask "1" --* "1" Task - -@enduml - -@startuml -'left to right direction -package "parallel" { - -object "Task.Body.invoke" as Task_Body - -class Task -class "Task.Signature" as Task_Signature -class ParallelStateFlow -interface Type -interface ExecuteTask -interface ParallelState - -'package "Factory" { -object "using" as usingType -object "using" as usingSignature -object from -usingSignature -right- usingType -usingType -right- from -'} -object "invoke" as invokeExecution { -execute tasks -} -object "invoke" as reduce { -reduce tasks -} - -Task_Body ..> ExecuteTask - -ExecuteTask <-- usingSignature -usingSignature --# Task_Signature -usingSignature ..> Task -usingSignature --* usingType - -Type #-- usingType -ExecuteTask <-- usingType -usingType ..> Task - -Type <-- from -Type #-- from -from ..> Task_Signature -from -* usingType - -reduce <- Task -reduce .> Task - -Task --> invokeExecution -invokeExecution <- ParallelState -invokeExecution ..> ParallelStateFlow - -} -@enduml diff --git a/docs/hld/parallel-execution.puml b/docs/hld/parallel-execution.puml index 11080238d5..4af6b84e51 100644 --- a/docs/hld/parallel-execution.puml +++ b/docs/hld/parallel-execution.puml @@ -43,12 +43,12 @@ detach split again -[hidden]-> partition async { -:log started; +:log start; :prepare context; -if (execute task in context) -:log succeed; -else -:log failed; +if (execute task in context) then (try) +:log success; +else (catch) +:log exception; endif :send result] detach diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index 36562a1d54..306cbe44b7 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -62,3 +62,7 @@ The following diagram is showing parallel execution algorithm in details: * is storing the result from each executed task. * is used for providing arguments for tasks. * is emitted in flow after each change. + +## API + +![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution-api.puml) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt index 91b83827cd..5374b6b5c1 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt @@ -39,4 +39,4 @@ infix fun Parallel.Type.using( internal fun validator( context: (() -> C) ): Parallel.Task = - context() using { context().also { it.state = this }.run { eager() } } + context() using { context().also { it.state = this }.run { validate() } } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt index 011f49ed67..d3943917d2 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -62,19 +62,24 @@ object Parallel { /** * Eager property delegate provider and initializer of a state values. */ - val eager = EagerProperties { state } + protected val eager = EagerProperties { state } /** * DSL for creating eager delegate accessor to the state value for a given type. * * @see [EagerProperties.invoke] */ - inline operator fun Type.not() = eager(this) + protected inline operator fun Type.not() = eager(this) /** * DSL for creating lazy delegate accessor to the state value for a given type. */ - operator fun Type.unaryMinus() = lazyProperty(this) + protected operator fun Type.unaryMinus() = lazyProperty(this) + + /** + * Internal accessor for initializing (validating) eager properties + */ + internal fun validate() = eager() } /** From aa66515536f44a35ea2b1b8f5ee84d63da9155a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Wed, 9 Jun 2021 22:06:45 +0200 Subject: [PATCH 07/29] Update --- docs/hld/parallel-execution-api-functions.puml | 12 ++++++------ docs/hld/parallel-execution-api-structures.puml | 8 ++++---- tool/execution/parallel/README.md | 5 +++-- .../kotlin/flank/exection/parallel/Parallel.kt | 11 ++++++----- .../exection/parallel/internal/ContextProvider.kt | 13 +++++++++++++ .../parallel/internal/CreateTaskFunction.kt | 14 -------------- .../test/kotlin/flank/exection/parallel/Example.kt | 12 ++++++------ .../kotlin/flank/exection/parallel/ParallelTest.kt | 4 ++-- 8 files changed, 40 insertions(+), 39 deletions(-) create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt delete mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/CreateTaskFunction.kt diff --git a/docs/hld/parallel-execution-api-functions.puml b/docs/hld/parallel-execution-api-functions.puml index 1f2f08d87b..a5cd57e482 100644 --- a/docs/hld/parallel-execution-api-functions.puml +++ b/docs/hld/parallel-execution-api-functions.puml @@ -2,14 +2,14 @@ 'left to right direction package "parallel" { -object "Task.Body.invoke()" as Task_Body { +object "Parallel.Function.invoke()" as Function { creates } -class Task -class "Task.Signature" as Task_Signature -interface Type -interface "Type" as Type2 +class "Parallel.Task" as Task +class "Parallel.Task.Signature" as Task_Signature +interface "Parallel.Type" as Type +interface "Parallel.Type" as Type2 interface ExecuteTask interface ParallelState @@ -30,7 +30,7 @@ object "invoke()" as reduce { reduces } -Task_Body "1" ..> ExecuteTask +Function "1" ..> ExecuteTask ExecuteTask <--o "1" usingSignature usingSignature "1" #--> Task_Signature diff --git a/docs/hld/parallel-execution-api-structures.puml b/docs/hld/parallel-execution-api-structures.puml index 28bf68be8b..0f0fa7aa42 100644 --- a/docs/hld/parallel-execution-api-structures.puml +++ b/docs/hld/parallel-execution-api-structures.puml @@ -14,11 +14,11 @@ class "Parallel.Task.Signature" as Task_Signature{ returns: Type args: Set> } -class "Parallel.Task.Body" as Task_Body { -= (X.() -> R) -> ParallelState.() -> R +class "Parallel.Function" as Function { += suspend (X.() -> R) -> ExecuteTask } interface ExecuteTask << (T, orchid) >> { -= ParallelState.() -> R += suspend ParallelState.() -> R } interface "Parallel.Type" as Type interface ParallelState << (T, orchid) >> { @@ -36,7 +36,7 @@ Type "1" --* "1" Task_Signature Type "*" --o "1" Task_Signature Task_Signature "1" --* "*" Task -Task_Body "1" ..> "*" ExecuteTask +Function "1" ..> "*" ExecuteTask ExecuteTask "1" --* "1" Task @enduml diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index 306cbe44b7..bf4eb7a23d 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -47,7 +47,7 @@ The following diagram is showing parallel execution algorithm in details: * is a set of `argument` `types` related with one `return` `type` just like in normal functions. -### Function +### Function (ExecuteTask) * is getting access to arguments through reduced copy of state. * is returning object described by the return type. @@ -65,4 +65,5 @@ The following diagram is showing parallel execution algorithm in details: ## API -![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution-api.puml) +![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution-api-functions.puml) +![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution-api-structures.puml) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt index d3943917d2..e1cb284a3d 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -1,6 +1,6 @@ package flank.exection.parallel -import flank.exection.parallel.internal.CreateTaskFunction +import flank.exection.parallel.internal.ContextProvider import flank.exection.parallel.internal.EagerProperties import flank.exection.parallel.internal.Execution import flank.exection.parallel.internal.initialValidators @@ -102,12 +102,13 @@ object Parallel { val args: Set>, ) - /** - * Parameterized factory for creating task functions in scope of [X]. - */ - class Body(override val context: () -> X) : CreateTaskFunction } + /** + * Parameterized factory for creating task functions in scope of [X]. + */ + class Function(override val context: () -> X) : ContextProvider + data class Event internal constructor( val type: Type<*>, val data: Any, diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt new file mode 100644 index 0000000000..587ed58032 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt @@ -0,0 +1,13 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.ExecuteTask +import flank.exection.parallel.Parallel + +/** + * Abstract factory for creating task function. + */ +internal interface ContextProvider { + val context: () -> X + operator fun invoke(body: suspend X.() -> R): ExecuteTask = + { context().also { it.state = this }.body() } +} diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/CreateTaskFunction.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/CreateTaskFunction.kt deleted file mode 100644 index 0ac12e33e3..0000000000 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/CreateTaskFunction.kt +++ /dev/null @@ -1,14 +0,0 @@ -package flank.exection.parallel.internal - -import flank.exection.parallel.Parallel -import flank.exection.parallel.ParallelState - -/** - * Abstract factory for creating task function. - */ -internal interface CreateTaskFunction { - val context: () -> X - operator fun invoke(body: suspend X.() -> T): suspend ParallelState.() -> T = { - context().also { it.state = this }.run { body() } - } -} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt index 435a4e166a..295244d6d5 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt @@ -56,7 +56,7 @@ private object Example { /** * Factory method for creating task functions with [Context]. */ - val func = Parallel.Task.Body(::Context) + val context = Parallel.Function(::Context) /** * List of tasks in [Example] scope @@ -82,32 +82,32 @@ private object Example { "hello" } - private val helloFail = Hello.Failing from setOf(Hello) using func { + private val helloFail = Hello.Failing from setOf(Hello) using context { throw IllegalStateException(hello) } - private val produceA = A using func { + private val produceA = A using context { (0..args.a).asFlow().onEach { out("A" to it) delay((0..args.wait).random()) }.toList() } - private val produceB = B using func { + private val produceB = B using context { (0..args.b).asFlow().onEach { out("B" to it) delay((0..args.wait).random()) }.toList() } - private val produceC = C using func { + private val produceC = C using context { (0..args.c).asFlow().onEach { out("C" to it) delay((0..args.wait).random()) }.toList() } - private val groupAndCount = Summary from setOf(A, B, C) using func { + private val groupAndCount = Summary from setOf(A, B, C) using context { delay(args.wait) (a + b + c) .groupBy { it } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt index cec79b4e6d..d44ec8776f 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt @@ -146,9 +146,9 @@ class ParallelTest { val c by -C } - val func = Parallel.Task.Body(::Context) + val context = Parallel.Function(::Context) val execute = setOf( - A from setOf(B) using func { + A from setOf(B) using context { initial b c // accessing this property will throw exception From fdfe1829ac3b46e33f51d1a9e576a7e9170b6f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 10 Jun 2021 01:28:22 +0200 Subject: [PATCH 08/29] Update --- docs/hld/task-graph.puml | 74 +++++----- tool/execution/parallel/README.md | 126 +++++++++++++++--- .../exection/parallel/internal/Execution.kt | 2 +- 3 files changed, 145 insertions(+), 57 deletions(-) diff --git a/docs/hld/task-graph.puml b/docs/hld/task-graph.puml index 13c69f9a22..b5942067d7 100644 --- a/docs/hld/task-graph.puml +++ b/docs/hld/task-graph.puml @@ -29,61 +29,61 @@ I1 *--> I1 } } package "Selected graph" { -map H3 #lightgreen { +map H2 #lightgreen { } -map I3 #lightgreen { +map I2 #lightgreen { } -map F3 #lightgrey { -I3 *--> I3 -} -map E3 #lightgreen { -I3 *--> I3 -} -map D3 #lightgreen { -I3 *--> I3 -} -map C3 #lightgreen { -D3 *--> D3 -} -map B3 #lightblue { -C3 *-> C3 -E3 *--> E3 -H3*---> H3 -} -map A3 #lightgrey { -B3 *--> B3 -F3 *--> F3 -C3 *--> C3 -I3 *--> I3 -} -} -package "Broken graph" { -map H2 #lightgrey { -} -map I2 #black { -} -map F2 #red { +map F2 #lightgrey { I2 *--> I2 } -map E2 #red { +map E2 #lightgreen { I2 *--> I2 } -map D2 #red { +map D2 #lightgreen { I2 *--> I2 } -map C2 #yellow { +map C2 #lightgreen { D2 *--> D2 } -map B2 #yellow { +map B2 #lightblue { C2 *-> C2 E2 *--> E2 H2*---> H2 } -map A2 #yellow { +map A2 #lightgrey { B2 *--> B2 F2 *--> F2 C2 *--> C2 I2 *--> I2 } } +package "Broken graph" { +map H3 #lightgrey { +} +map I3 #black { +} +map F3 #red { +I3 *--> I3 +} +map E3 #red { +I3 *--> I3 +} +map D3 #red { +I3 *--> I3 +} +map C3 #yellow { +D3 *--> D3 +} +map B3 #yellow { +C3 *-> C3 +E3 *--> E3 +H3*---> H3 +} +map A3 #yellow { +B3 *--> B3 +F3 *--> F3 +C3 *--> C3 +I3 *--> I3 +} +} @enduml diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index bf4eb7a23d..a5fd606a26 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -13,40 +13,47 @@ Library for task parallelization and asynchronous execution. # Design -1. Imagine a complicated long-running `execution` that needs to perform several operations (`tasks`) in correct order to collect a required `data` and produce `side effects`. -2. Any `execution` like that, can be modeled as the set of unrelated data `types` and suspendable functions (`tasks`). - -## Execution - -* will run tasks in optimized order creating synchronization points where needed and running all other stuff in parallel. -* is collecting result from each task, and accumulating it to state. -* is returning the flow of realtime state changes. -* can detach and fail if set of `tasks` are creating broken graph with `missing dependencies`. -* Each task in execution scope must have unique return type. This is crucial for producing correct graph. - -The following diagram is showing parallel execution algorithm in details: - -![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution.puml) +Imagine a complicated long-running `execution` that needs to perform several operations (`tasks`) in correct order to collect a required `data` and produce `side effects`. Any `execution` like that, can be modeled as set of unrelated data `types` and suspendable functions (`tasks`). ## Type -* is atomic and unique in the execution scope. +* is unique in the execution scope. * is describing the data type for value that can be: - * passed to execution as arguments for initial state. + * passed to execution as arguments for initial state. * passed to task from state as argument. * returned from task. * returned from execution. +### Example + +```kotlin +object Args : Parallel.Type> +object Foo : Parallel.Type +object Bar : Parallel.Type +object Baz : Parallel.Type +object Logger : Parallel.Type +``` + ## Task * `arguments` for task are coming from `initial state` (`arguments`) or results from other `tasks`. * can be grouped into set to create the `execution` graph. * graph can have broken paths, when any task have `missing dependencies`. +* can be uniquely identified using return type of its signature. ### Signature * is a set of `argument` `types` related with one `return` `type` just like in normal functions. +#### Example + +```kotlin +// following expression +Foo from setOf(Bar, Baz) +// is equal to +Parallel.Task.Signature(returns = Foo, args = setOf(Bar, baz)) +``` + ### Function (ExecuteTask) * is getting access to arguments through reduced copy of state. @@ -54,16 +61,97 @@ The following diagram is showing parallel execution algorithm in details: ### Missing dependencies +* argument types that are not specified as return types by any task in the graph or provided in the initial state. * missing dependencies can be detected before execution from the set of `tasks` and `initial state`. +### Example + +```kotlin + +// Context with accessors to state values. +class Context : Parallel.Context() { + // Delegates specified with '!' are used for validating initial state. + // Are provided by default to each task. + val args by !Args + val out by !Logger + + // Dynamic properties for providing results from tasks as arguments for other tasks. + val foo by -Foo + val bar by -Bar + val baz by -Baz +} + +// Helper for creating task functions with context. +val context = Parallel.Function(::Context) + +// Set of tasks to execute. +val execute = setOf( + // Basic task without arguments and access to the context. + Baz using { false }, + // Task that uses context to access state values, initial properties doesn't need to be specified in arguments. + Bar using context { args.size }, + // Task that specifies required arguments and generates side effect by emitting value to output + Foo from setOf(Bar, Baz) using context { "$bar, $baz".apply(out) } +) +``` + +### Graph + +Task graphs can be represented as tree structure without cycles, where each type can reference others. + +![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/task-graph.puml) + ## State * is a map of types with associated values that are coming from initial arguments or task results. * is storing the result from each executed task. -* is used for providing arguments for tasks. -* is emitted in flow after each change. +* is used for providing arguments and dependencies for tasks. +* is emitted in flow after each change. + +### Example + +```kotlin +// Preparing initial state +val initial = mapOf( + Args to listOf(), + Logger to { any: Any -> println(any) }, +) +``` + +## Execution + +* will run tasks in optimized order creating synchronization points where needed and running all other stuff in parallel. +* is collecting result from each task, and accumulating it to state. +* is returning the flow of realtime state changes. +* can detach and fail if set of `tasks` are creating broken graph with `missing dependencies`. +* Each task in execution scope must have unique return type. This is crucial for producing correct graph. + +```kotlin -## API +// For execution without arguments +val flow1: Flow = execute() + +// For execution with arguments (initial state) +val flow2: Flow = execute(intial) + +// For execution reduced to selected types with dependencies. +val flow3: Flow = execute(Bar, Baz)(intial) +``` + +### Process + +The following diagram is showing parallel execution algorithm in details: + +![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution.puml) + +# Relations + +The following graphs are showing constrains between elements described previously. + +### Functions ![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution-api-functions.puml) + +### Types + ![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution-api-structures.puml) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt index 129055c4c0..4904e7a7de 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt @@ -60,7 +60,7 @@ internal operator fun Execution.invoke(): Flow = .drop(1) /** - * Internal context of the tasks execution. + * Internal context of the execution. A single instance can be invoked only once. */ internal class Execution( /** From bc0e85d663175e1bf8289d32ccff2b7867757c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 10 Jun 2021 01:58:57 +0200 Subject: [PATCH 09/29] Update --- docs/hld/task-graph.puml | 32 ++++++++++++++++++++++++++++++- tool/execution/parallel/README.md | 29 +++++++++++----------------- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/docs/hld/task-graph.puml b/docs/hld/task-graph.puml index b5942067d7..cb5d51ae98 100644 --- a/docs/hld/task-graph.puml +++ b/docs/hld/task-graph.puml @@ -58,7 +58,7 @@ I2 *--> I2 } } package "Broken graph" { -map H3 #lightgrey { +map H3 #lightgreen { } map I3 #black { } @@ -86,4 +86,34 @@ C3 *--> C3 I3 *--> I3 } } + +package "Failed graph" { +map H4 #lightgreen { +} +map I4 #lightgreen { +} +map F4 #red { +I4 *--> I4 +} +map E4 #yellow { +I4 *--> I4 +} +map D4 #yellow { +I4 *--> I4 +} +map C4 #lightgrey { +D4 *--> D4 +} +map B4 #lightgrey { +C4 *-> C4 +E4 *--> E4 +H4*---> H4 +} +map A4 #lightgrey { +B4 *--> B4 +F4 *--> F4 +C4 *--> C4 +I4 *--> I4 +} +} @enduml diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index a5fd606a26..db74218d4e 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -36,23 +36,14 @@ object Logger : Parallel.Type ## Task +* is representing a single atomic operation in the execution process. +* is a relation of signature and task function. * `arguments` for task are coming from `initial state` (`arguments`) or results from other `tasks`. -* can be grouped into set to create the `execution` graph. -* graph can have broken paths, when any task have `missing dependencies`. * can be uniquely identified using return type of its signature. ### Signature -* is a set of `argument` `types` related with one `return` `type` just like in normal functions. - -#### Example - -```kotlin -// following expression -Foo from setOf(Bar, Baz) -// is equal to -Parallel.Task.Signature(returns = Foo, args = setOf(Bar, baz)) -``` +* is a set of argument `types` related with one return `type` just like in normal functions. ### Function (ExecuteTask) @@ -64,6 +55,14 @@ Parallel.Task.Signature(returns = Foo, args = setOf(Bar, baz)) * argument types that are not specified as return types by any task in the graph or provided in the initial state. * missing dependencies can be detected before execution from the set of `tasks` and `initial state`. +### Graph + +* can be represented as tree structure without cycles, where each type can reference others. +* can be created from a set of tasks. +* can have broken paths, when any task have `missing dependencies`. + +![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/task-graph.puml) + ### Example ```kotlin @@ -95,12 +94,6 @@ val execute = setOf( ) ``` -### Graph - -Task graphs can be represented as tree structure without cycles, where each type can reference others. - -![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/task-graph.puml) - ## State * is a map of types with associated values that are coming from initial arguments or task results. From 90e95afbaf3a92571d475c96909ac65c7b1cc6c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Thu, 10 Jun 2021 20:30:58 +0200 Subject: [PATCH 10/29] Improve graph validation --- .../parallel/{Factory.kt => Create.kt} | 0 .../kotlin/flank/exection/parallel/Execute.kt | 32 ++++ .../kotlin/flank/exection/parallel/Ext.kt | 25 --- .../flank/exection/parallel/Parallel.kt | 57 ++---- .../kotlin/flank/exection/parallel/Reduce.kt | 23 +++ .../flank/exection/parallel/Validate.kt | 43 +++++ .../exection/parallel/internal/Execution.kt | 10 +- .../flank/exection/parallel/internal/Ext.kt | 6 + .../exection/parallel/internal/Prepare.kt | 2 +- .../exection/parallel/internal/Reduce.kt | 10 +- .../exection/parallel/internal/Validate.kt | 36 ---- .../parallel/internal/graph/FindCycles.kt | 56 ++++++ .../graph/FindDuplicatedDependencies.kt | 9 + .../internal/graph/FindMissingDependencies.kt | 8 + .../kotlin/flank/exection/parallel/Example.kt | 12 +- .../{ParallelTest.kt => ExecuteKtTest.kt} | 20 +- .../flank/exection/parallel/ReduceKtTest.kt | 42 +++++ .../flank/exection/parallel/ValidateKtTest.kt | 171 ++++++++++++++++++ .../parallel/internal/ReduceKtTest.kt | 44 ----- .../parallel/internal/ValidateKtTest.kt | 46 ----- .../internal/graph/FindCyclesKtTest.kt | 131 ++++++++++++++ 21 files changed, 561 insertions(+), 222 deletions(-) rename tool/execution/parallel/src/main/kotlin/flank/exection/parallel/{Factory.kt => Create.kt} (100%) create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt delete mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Validate.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Ext.kt delete mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindCycles.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindDuplicatedDependencies.kt create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindMissingDependencies.kt rename tool/execution/parallel/src/test/kotlin/flank/exection/parallel/{ParallelTest.kt => ExecuteKtTest.kt} (88%) create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ReduceKtTest.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ValidateKtTest.kt delete mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt delete mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Create.kt similarity index 100% rename from tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Factory.kt rename to tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Create.kt diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt new file mode 100644 index 0000000000..b96b235475 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt @@ -0,0 +1,32 @@ +package flank.exection.parallel + +import flank.exection.parallel.internal.Execution +import flank.exection.parallel.internal.invoke +import kotlinx.coroutines.flow.Flow + +/** + * Execute tasks in parallel with a given args. + * Before executing, the [validate] is performed on a [Tasks] for a given [args] to detect missing dependencies. + * + * @return [Flow] of [ParallelState] changes. + */ +infix operator fun Tasks.invoke( + args: ParallelState +): Flow = + Execution(this, args).invoke() + + +// ======================= Extensions ======================= + +/** + * Shortcut for executing tasks on a given state. + */ +operator fun Tasks.invoke( + vararg state: Pair, Any> +): Flow = invoke(state.toMap()) + +/** + * Shortcut for executing tasks on empty state. + */ +operator fun Tasks.invoke(): Flow = + invoke(emptyMap()) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt deleted file mode 100644 index e5f17ee202..0000000000 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Ext.kt +++ /dev/null @@ -1,25 +0,0 @@ -package flank.exection.parallel - -import kotlinx.coroutines.flow.Flow - -// ======================= Reduce ======================= -/** - * Shortcut for tasks reducing. - */ -operator fun Tasks.invoke( - vararg returns: Parallel.Type<*> -): Tasks = invoke(returns.toSet()) - -// ======================= Execute ======================= -/** - * Shortcut for executing tasks on a given state. - */ -operator fun Tasks.invoke( - vararg state: Pair, Any> -): Flow = invoke(state.toMap()) - -/** - * Shortcut for executing tasks on empty state. - */ -operator fun Tasks.invoke(): Flow = - invoke(emptyMap()) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt index e1cb284a3d..23d21bc289 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -2,40 +2,9 @@ package flank.exection.parallel import flank.exection.parallel.internal.ContextProvider import flank.exection.parallel.internal.EagerProperties -import flank.exection.parallel.internal.Execution -import flank.exection.parallel.internal.initialValidators -import flank.exection.parallel.internal.invoke import flank.exection.parallel.internal.lazyProperty -import flank.exection.parallel.internal.reduceTo -import flank.exection.parallel.internal.validate -import kotlinx.coroutines.flow.Flow import java.lang.System.currentTimeMillis -// ======================= Core functions ======================= - -/** - * Reduce given [Tasks] by [select] types to remove unneeded tasks from the graph. - * The returned graph will only tasks that are returning selected types, their dependencies and derived dependencies. - * Additionally this is keeping also the validators for initial state. - * - * @return Reduced [Tasks] - */ -operator fun Tasks.invoke( - select: Set> -): Tasks = - reduceTo(select + initialValidators) - -/** - * Execute tasks in parallel with a given args. - * Before executing, the [validate] is performed on a [Tasks] for a given [args] to detect missing dependencies. - * - * @return [Flow] of [ParallelState] changes. - */ -infix operator fun Tasks.invoke( - args: ParallelState -): Flow = - Execution(validate(args), args).invoke() - // ======================= Types ======================= /** @@ -95,13 +64,15 @@ object Parallel { val execute: ExecuteTask ) { /** - * The task signature that contains arguments types and returned type. + * The task signature. + * + * @param type A return type of a task + * @param args A set of types for arguments */ data class Signature( - val returns: Type, + val type: Type, val args: Set>, ) - } /** @@ -120,9 +91,14 @@ object Parallel { object Logger : Type - data class ValidationError( - val data: MissingDependencies - ) : Exception("Cannot resolve dependencies $data") + object DependenciesError { + data class Missing(val data: TypeDependencies) : Error("Missing dependencies $data") + data class Duplicate(val data: DuplicateDependencies) : Error("Duplicate dependencies $data") + data class NoEntryPoint(val initial: Set>, val tasks: TypeDependencies) : + Error("No entry points in tasks $tasks with initial state $initial") + + data class Cycles(val data: List>>) : Error("Found cycles in graph $data") + } } // ======================= Aliases ======================= @@ -150,4 +126,9 @@ typealias Tasks = Set> /** * The [Map.values] are representing missing dependencies required by the tasks to provide [Map.keys]. */ -typealias MissingDependencies = Map, Set>> +typealias TypeDependencies = Map, Set>> + +/** + * The [Map.values] are representing missing dependencies required by the tasks to provide [Map.keys]. + */ +typealias DuplicateDependencies = Map, Int> diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt new file mode 100644 index 0000000000..b8d52661e6 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt @@ -0,0 +1,23 @@ +package flank.exection.parallel + +import flank.exection.parallel.internal.initialValidators +import flank.exection.parallel.internal.reduceTo + +/** + * Reduce given [Tasks] by [select] types to remove unneeded tasks from the graph. + * The returned graph will only tasks that are returning selected types, their dependencies and derived dependencies. + * Additionally this is keeping also the validators for initial state. + * + * @return Reduced [Tasks] + */ +operator fun Tasks.invoke( + select: Set> +): Tasks = + reduceTo(select + initialValidators) + +/** + * Shortcut for tasks reducing. + */ +operator fun Tasks.invoke( + vararg returns: Parallel.Type<*> +): Tasks = invoke(returns.toSet()) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Validate.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Validate.kt new file mode 100644 index 0000000000..a47694d143 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Validate.kt @@ -0,0 +1,43 @@ +package flank.exection.parallel + +import flank.exection.parallel.internal.args +import flank.exection.parallel.internal.graph.findCycles +import flank.exection.parallel.internal.graph.findDuplicatedDependencies +import flank.exection.parallel.internal.graph.findMissingDependencies +import flank.exection.parallel.internal.type +import kotlinx.coroutines.runBlocking + +/** + * Validate the given [Tasks] and [ParallelState] for finding missing dependencies or broken paths. + * + * @param initial The initial arguments for tasks execution. + * @return Valid [Tasks] if graph has no broken paths or missing dependencies. + */ +fun Tasks.validate(initial: ParallelState = emptyMap()): Tasks = run { + // Separate initial validators from tasks. Validators are important now but not during the execution. + val (validators, tasks) = splitTasks() + + // check if initial state is providing all required values specified in context. + runBlocking { validators.forEach { check -> check.execute(initial) } } + + map(Parallel.Task<*>::type).findDuplicatedDependencies(initial.keys).run { + if (isNotEmpty()) throw Parallel.DependenciesError.Duplicate(this) + } + + val graph = associate { task -> task.type to task.args } + + graph.findMissingDependencies(initial.keys).run { + if (isNotEmpty()) throw Parallel.DependenciesError.Missing(this) + } + + graph.findCycles().run { + if (isNotEmpty()) throw Parallel.DependenciesError.Cycles(this) + } + + tasks.toSet() +} + +private fun Iterable>.splitTasks() = this + .groupBy { task -> task.type is Parallel.Context } + .run { getOrDefault(true, emptyList()) to getOrDefault(false, emptyList()) } + diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt index 4904e7a7de..ee70ab04e5 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt @@ -90,12 +90,12 @@ internal class Execution( /** * The set of value types required to complete the execution. */ - val required = tasks.map { task -> task.signature.returns }.toSet() + val required = tasks.map(Task<*>::type).toSet() /** * Map of remaining tasks for run grouped by arguments. */ - val remaining = tasks.groupBy { task -> task.signature.args }.toMutableMap() + val remaining = tasks.groupBy(Task<*>::args).toMutableMap() /** * Reference to optional output for structural logs. @@ -123,7 +123,7 @@ private suspend fun Execution.abortBy(type: Type<*>, cause: Throwable) { // Add remaining tasks to state. remaining.toMap().values.flatten().forEach { task -> - channel.trySend(task.signature.returns to task) + channel.trySend(task.type to task) } // Close execution channel. @@ -185,8 +185,8 @@ private fun Execution.execute( task: Task<*>, args: Map, Any>, ) { - // Obtain return type for current task. - val type: Type<*> = task.signature.returns + // Obtain type of current task. + val type: Type<*> = task.type // Keep task job. jobs += type to scope.launch { diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Ext.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Ext.kt new file mode 100644 index 0000000000..53a2e7cc5c --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Ext.kt @@ -0,0 +1,6 @@ +package flank.exection.parallel.internal + +import flank.exection.parallel.Parallel + +internal val Parallel.Task.type get() = signature.type +internal val Parallel.Task.args get() = signature.args diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt index dc9c3b6efc..4aab7b76f7 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt @@ -8,4 +8,4 @@ import flank.exection.parallel.Tasks * This is necessary to perform validations of initial state before the execution. */ internal val Tasks.initialValidators: List - get() = mapNotNull { task -> task.signature.returns as? Parallel.Context } + get() = mapNotNull { task -> task.type as? Parallel.Context } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt index 18d23d2213..083dc6745f 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt @@ -6,8 +6,12 @@ import flank.exection.parallel.Tasks internal infix fun Tasks.reduceTo( selectedTypes: Set> ): Tasks = - filter { task -> task.signature.run { returns in selectedTypes } } + filter { task -> task.type in selectedTypes } .toSet() + .apply { + val notFound = selectedTypes - map(Parallel.Task<*>::type) + if (notFound.isNotEmpty()) throw Exception("Cannot reduce find tasks for the following types: $notFound") + } .reduce(this) /** @@ -26,7 +30,7 @@ private tailrec fun Tasks.reduce( ): Tasks = when { isEmpty() -> acc - else -> flatMap { task -> task.signature.args } + else -> flatMap(Parallel.Task<*>::args) .mapNotNull(all::findByReturnType) .toSet() .reduce(all, acc + this) @@ -35,4 +39,4 @@ private tailrec fun Tasks.reduce( private fun Tasks.findByReturnType( type: Parallel.Type<*> ): Parallel.Task<*>? = - find { task -> task.signature.returns == type } + find { task -> task.type == type } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt deleted file mode 100644 index 585c6fd463..0000000000 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Validate.kt +++ /dev/null @@ -1,36 +0,0 @@ -package flank.exection.parallel.internal - -import flank.exection.parallel.MissingDependencies -import flank.exection.parallel.Parallel -import flank.exection.parallel.ParallelState -import flank.exection.parallel.Tasks -import kotlinx.coroutines.runBlocking - -/** - * Validate the given [Tasks] and [ParallelState] for finding missing dependencies or broken paths. - * - * @param initial The initial arguments for tasks execution. - * @return Valid [Tasks] if graph has no broken paths or missing dependencies. - * @throws [Parallel.ValidationError] if find broken paths between tasks. - */ -internal fun Tasks.validate(initial: ParallelState = emptyMap()): Tasks = run { - val (validators, tasks) = splitTasks() - - // check if initial state is providing all required values specified in context. - runBlocking { validators.forEach { check -> check.execute(initial) } } - - // validate graph is not broken - findMissingDependencies(initial.keys).run { if (isNotEmpty()) throw Parallel.ValidationError(this) } - - tasks.toSet() -} - -private fun Iterable>.splitTasks() = this - .groupBy { it.signature.returns is Parallel.Context } - .run { getOrDefault(true, emptyList()) to getOrDefault(false, emptyList()) } - -private fun Tasks.findMissingDependencies(initial: Set>): MissingDependencies { - val all = map { task -> task.signature.returns }.toSet() + initial - return associate { task -> task.signature.run { returns to args - all } } - .filterValues { missing -> missing.isNotEmpty() } -} diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindCycles.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindCycles.kt new file mode 100644 index 0000000000..73d079535c --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindCycles.kt @@ -0,0 +1,56 @@ +package flank.exection.parallel.internal.graph + +tailrec fun Map>.findCycles( + + remaining: List = toList() + .sortedByDescending { (_, edges) -> edges.size } + .map { (vertices, _) -> vertices } + .toMutableList(), + + queue: List = emptyList(), + + path: List = emptyList(), + + visited: Set = emptySet(), + + cycles: List> = emptyList() + +): List> { + + val current: T = queue.firstOrNull() + ?: remaining.firstOrNull() + ?: return cycles + + val next: List = getOrDefault(current, emptySet()) + .intersect(path + remaining + queue) + .toList() + + val cycle = current in path + next + + // println("$cycle R:$remaining Q:$queue N:$next C:$current P:$path V:$visited") + // println() + + return findCycles( + remaining = + remaining - (next + current), + + queue = + if (cycle) queue + else (next + queue) - current, + + path = + when { + cycle -> emptyList() + next.isEmpty() -> emptyList() + else -> path + current + }, + + visited = + if (cycle || queue.isEmpty()) visited + path + else visited, + + cycles = + if (cycle) cycles.plus(element = path + current) + else cycles + ) +} diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindDuplicatedDependencies.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindDuplicatedDependencies.kt new file mode 100644 index 0000000000..242c9942ec --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindDuplicatedDependencies.kt @@ -0,0 +1,9 @@ +package flank.exection.parallel.internal.graph + +internal fun List.findDuplicatedDependencies( + initial: Set +): Map = + plus(initial) + .groupBy { it } + .mapValues { (_, list) -> list.size - 1 } + .filterValues { it > 0 } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindMissingDependencies.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindMissingDependencies.kt new file mode 100644 index 0000000000..5e281e3212 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindMissingDependencies.kt @@ -0,0 +1,8 @@ +package flank.exection.parallel.internal.graph + +internal fun Map>.findMissingDependencies( + initial: Set +): Map> { + val all = keys + initial + return mapValues { (_, args) -> args - all }.filterValues { missing -> missing.isNotEmpty() } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt index 295244d6d5..98bd90fd55 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt @@ -53,11 +53,6 @@ private object Example { object Failing : Type } - /** - * Factory method for creating task functions with [Context]. - */ - val context = Parallel.Function(::Context) - /** * List of tasks in [Example] scope */ @@ -73,7 +68,12 @@ private object Example { ) } - // ======================= Internal Tasks ======================= + /** + * Factory method for creating task functions with [Context]. + */ + private val context = Parallel.Function(::Context) + + // ======================= Tasks ======================= private val validate: Parallel.Task = validator(::Context) diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt similarity index 88% rename from tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt rename to tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt index d44ec8776f..a849852ca4 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ParallelTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt @@ -10,7 +10,6 @@ import kotlinx.coroutines.withTimeout import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Test -import java.lang.ClassCastException import java.util.concurrent.atomic.AtomicInteger // ====================== COMMON TYPES ====================== @@ -24,7 +23,7 @@ object E : Parallel.Type // ====================== TESTS ====================== -class ParallelTest { +class ExecuteKtTest { /** * Executing [Tasks] will return empty flow of [ParallelState]. @@ -128,7 +127,7 @@ class ParallelTest { val execute = setOf( A using { delay(50); throw Exception() }, B using { delay(100) }, - C from setOf(B) using {} + C from setOf(B) using { } ) val actual = runBlocking { execute().last() } assertTrue(actual[B].toString(), actual[B] is CancellationException) @@ -163,19 +162,4 @@ class ParallelTest { assert(actual[A] is NullPointerException) } - - /** - * If the validator function is specified in tasks it will validate arguments before running the execution. - * The failing validation will throw [NullPointerException] or [ClassCastException] - */ - @Test(expected = ClassCastException::class) - fun `validate arguments`() { - class Context : Parallel.Context() { - val initial by !IntType - } - - val args: ParallelState = mapOf(IntType to "asd") - val execute = setOf(validator(::Context)) - runBlocking { execute(args).last() } - } } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ReduceKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ReduceKtTest.kt new file mode 100644 index 0000000000..d5d2314b0e --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ReduceKtTest.kt @@ -0,0 +1,42 @@ +package flank.exection.parallel + +import flank.exection.parallel.internal.type +import org.junit.Assert +import org.junit.Test + +class ReduceKtTest { + + /** + * Select task with dependencies, other tasks will be filtered out. + */ + @Test + fun `select tasks`() { + val execute = setOf( + A from setOf(B, C) using { }, + B from setOf(D, E) using { }, + C using { }, + D using { }, + E using { }, + ) + + val expected = setOf(B, D, E) + + val actual = execute.invoke(B).map { it.type }.toSet() + + Assert.assertEquals(expected, actual) + } + + /** + * Selecting a type of missing task not specified in execution will throw exception. + */ + @Test(expected = Exception::class) + fun `select missing tasks`() { + val execute: Tasks = setOf(C using {}) + + val expected: Tasks = emptySet() + + val actual = execute.invoke(A, B).map { it.type }.toSet() + + Assert.assertEquals(expected, actual) + } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ValidateKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ValidateKtTest.kt new file mode 100644 index 0000000000..1bab556b9b --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ValidateKtTest.kt @@ -0,0 +1,171 @@ +package flank.exection.parallel + +import org.junit.Assert.* +import org.junit.Test +import java.lang.NullPointerException + +class ValidateKtTest { + + /** + * Validating proper execution will just return it. + */ + @Test + fun `validate proper execution`() { + val expected = setOf( + A from setOf(B) using { }, + B from setOf(C, D) using { }, + C from setOf(D) using { }, + D using { } + ) + assertEquals( + expected, + expected.validate(), + ) + } + + /** + * If the validator function is specified in tasks it will validate arguments. + * The failing validation will throw [NullPointerException] + */ + @Test(expected = NullPointerException::class) + fun `validate missing argument`() { + class Context : Parallel.Context() { + val initial by !IntType + } + + val execute = setOf(validator(::Context)) + execute.validate() + } + + /** + * If the validator function is specified in tasks it will validate arguments. + * The failing validation will throw [ClassCastException] + */ + @Test(expected = ClassCastException::class) + fun `validate argument type`() { + class Context : Parallel.Context() { + val initial by !IntType + } + + val args: ParallelState = mapOf(IntType to "asd") + val execute = setOf(validator(::Context)) + execute.validate(args) + } + + /** + * Running graph with unresolved dependencies should throw [Parallel.DependenciesError.Missing]. + */ + @Test + fun `missing dependencies`() { + val execute = setOf( + A from setOf(B, C) using { }, + D from setOf(E, C) using { }, + ) + val expected = mapOf( + A to setOf(B, C), + D to setOf(E, C), + ) + try { + execute.validate() + fail() + } catch (error: Parallel.DependenciesError.Missing) { + assertEquals(expected, error.data) + } + } + + /** + * Detect duplicated return values of tasks. + */ + @Test + fun `duplicate dependencies 1`() { + val execute = setOf( + A from setOf(B) using { }, + A from setOf(C) using { }, + A from setOf(D) using { }, + B from setOf(D) using { }, + B from setOf(C) using { }, + D using {}, + C using {} + ) + val expected = mapOf( + A to 2, + B to 1, + ) + try { + execute.validate() + fail() + } catch (error: Parallel.DependenciesError.Duplicate) { + assertEquals(expected, error.data) + } + } + + /** + * Detect duplicated return values of state tasks. + */ + @Test + fun `duplicate dependencies 2`() { + val args: ParallelState = mapOf( + A to Unit, + ) + val execute = setOf( + A from setOf(B) using { }, + A from setOf(C) using { }, + B using {}, + C using {}, + ) + val expected = mapOf( + A to 2, + ) + try { + execute.validate(args) + fail() + } catch (error: Parallel.DependenciesError.Duplicate) { + assertEquals(expected, error.data) + } + } + + @Test + fun `cyclic dependencies 1`() { + val execute = setOf( + A from setOf(A) using { }, + ) + val expected = listOf(listOf(A)) + try { + execute.validate() + fail() + } catch (e: Parallel.DependenciesError.Cycles) { + assertEquals(expected, e.data) + } + } + + @Test + fun `cyclic dependencies 2`() { + val execute = setOf( + A from setOf(B) using { }, + B from setOf(A) using { }, + ) + val expected = listOf(listOf(A, B, A)) + try { + execute.validate() + fail() + } catch (e: Parallel.DependenciesError.Cycles) { + assertEquals(expected, e.data) + } + } + + @Test + fun `cyclic dependencies 3`() { + val execute = setOf( + A from setOf(B) using { }, + B from setOf(C) using { }, + C from setOf(A) using { }, + ) + val expected = listOf(listOf(A, B, C, A)) + try { + execute.validate() + fail() + } catch (e: Parallel.DependenciesError.Cycles) { + assertEquals(expected, e.data) + } + } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt deleted file mode 100644 index ca2e39f4ce..0000000000 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ReduceKtTest.kt +++ /dev/null @@ -1,44 +0,0 @@ -package flank.exection.parallel.internal - -import flank.exection.parallel.Parallel.Type -import flank.exection.parallel.from -import flank.exection.parallel.using -import org.junit.Test - -class ReduceKtTest { - - private object A : Type - private object B : Type - private object C : Type - private object D : Type - private object E : Type - private object F : Type - - private val execute = setOf( - A using { }, - B using { }, - C from setOf(B) using { }, - D from setOf(A, C) using { }, - E using { }, - F from setOf(E) using { }, - ) - - @Test - fun test() { - val omitted = setOf(E, F) - val expected = execute.filter { it.signature.returns !in omitted } - - val actual = execute.reduceTo(setOf(D)) - - assert(expected.containsAll(actual)) - assert(expected.size == actual.size) - } - - @Test - fun test2() { - - val actual = execute.filter { it.signature.returns != C }.toSet().reduceTo(setOf(D)) - - actual.forEach(::println) - } -} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt deleted file mode 100644 index c599612fb2..0000000000 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/ValidateKtTest.kt +++ /dev/null @@ -1,46 +0,0 @@ -package flank.exection.parallel.internal - -import flank.exection.parallel.Parallel -import flank.exection.parallel.from -import flank.exection.parallel.using -import org.junit.Assert.assertEquals -import org.junit.Test - -class ValidateKtTest { - - private object A : Parallel.Type - private object B : Parallel.Type - private object C : Parallel.Type - private object D : Parallel.Type - - private val execute = setOf( - A using { }, - B from setOf(A) using { }, - C from setOf(B) using { }, - D from setOf(A, C) using { }, - ) - - /** - * Return empty map when there are no missing dependencies. - */ - @Test - fun test() { - execute.validate() - } - - /** - * Return missing map of dependencies dependencies - */ - @Test - fun test2() { - val expected = mapOf( - B to setOf(A), - D to setOf(A) - ) - try { - execute.drop(1).toSet().validate() - } catch (e: Parallel.ValidationError) { - assertEquals(expected, e.data) - } - } -} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt new file mode 100644 index 0000000000..9e61db8c3e --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt @@ -0,0 +1,131 @@ +package flank.exection.parallel.internal.graph + +import org.junit.Assert.* +import org.junit.Test + +class FindCyclesKtTest { + + + @Test + fun valid0() { + val graph = mapOf( + 0 to setOf(1, 2), + 1 to setOf(3), + 2 to setOf(3), + 3 to setOf(), + ) + + val actual = graph.findCycles() + + assertEquals(emptyList(), actual) + } + + @Test + fun validRandom0() = repeat(100) { + val nodes = (0..100) + val used = mutableSetOf() + val graph = nodes.associateWith { + used += it + (nodes.shuffled() - used).run { take((0..(size/10)).random()) }.toSet() + } + // println("=================================") + // graph.forEach { println(it) } + // println() + val actual = graph.findCycles() + + assertEquals(emptyList(), actual) + } + + @Test + fun cyclic0() { + val graph = mapOf( + 0 to setOf(0) + ) + val expected = listOf(listOf(0)) + + val actual = graph.findCycles() + + assertEquals(expected, actual) + } + + @Test + fun cyclic1() { + val graph = mapOf( + 0 to setOf(1), + 1 to setOf(0), + ) + val expected = listOf( + listOf(0, 1, 0) + ) + val actual = graph.findCycles() + + assertEquals(expected, actual) + } + + + @Test + fun cyclic2() { + val graph = mapOf( + 0 to setOf(1), + 1 to setOf(2), + 2 to setOf(0), + ) + val expected = listOf( + listOf(0, 1, 2, 0) + ) + val actual = graph.findCycles() + + assertEquals(expected, actual) + } + + @Test + fun cyclic3() { + val graph = mapOf( + 0 to setOf(1), + 1 to setOf(2), + 2 to setOf(3), + 3 to setOf(0), + ) + val expected = listOf( + listOf(0, 1, 2, 3, 0) + ) + val actual = graph.findCycles() + + assertEquals(expected, actual) + } + + @Test + fun cyclic4() { + val graph = mapOf( + 0 to setOf(1), + 1 to setOf(2), + 2 to setOf(3), + 3 to setOf(1), + ) + val expected = listOf( + listOf(0, 1, 2, 3, 1) + ) + val actual = graph.findCycles() + + assertEquals(expected, actual) + } + + @Test + fun cyclic5() { + val graph = mapOf( + 0 to setOf(1), + 1 to setOf(2), + 2 to setOf(3), + 3 to setOf(1, 4), + 4 to setOf(5), + 5 to setOf(3) + ) + val expected = listOf( + listOf(3,1,2,3), + listOf(3,4,5,3), + ) + val actual = graph.findCycles() + + assertEquals(expected, actual) + } +} From 71c4277553e0cbb7023f3f849f7064c5e529034d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 08:47:38 +0200 Subject: [PATCH 11/29] Remove unneeded error type --- .../src/main/kotlin/flank/exection/parallel/Parallel.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt index 23d21bc289..625306397c 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -94,9 +94,6 @@ object Parallel { object DependenciesError { data class Missing(val data: TypeDependencies) : Error("Missing dependencies $data") data class Duplicate(val data: DuplicateDependencies) : Error("Duplicate dependencies $data") - data class NoEntryPoint(val initial: Set>, val tasks: TypeDependencies) : - Error("No entry points in tasks $tasks with initial state $initial") - data class Cycles(val data: List>>) : Error("Found cycles in graph $data") } } From 23d972f83e1e826cc15d37c473a1da45949f4b7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 08:52:33 +0200 Subject: [PATCH 12/29] Rollback gradle version --- flank-scripts/gradle/wrapper/gradle-wrapper.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- test_runner/gradle/wrapper/gradle-wrapper.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flank-scripts/gradle/wrapper/gradle-wrapper.properties b/flank-scripts/gradle/wrapper/gradle-wrapper.properties index e85b8e3289..7fb030ba10 100644 --- a/flank-scripts/gradle/wrapper/gradle-wrapper.properties +++ b/flank-scripts/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index e85b8e3289..7fb030ba10 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/test_runner/gradle/wrapper/gradle-wrapper.properties b/test_runner/gradle/wrapper/gradle-wrapper.properties index c047c05416..3ff35d4a25 100644 --- a/test_runner/gradle/wrapper/gradle-wrapper.properties +++ b/test_runner/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-2-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-1-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From c202cdf60acabd8fbdeaf6f6222a38c7cdcc4f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 09:13:05 +0200 Subject: [PATCH 13/29] Format kotlin --- .../src/main/kotlin/flank/exection/parallel/Execute.kt | 1 - .../src/main/kotlin/flank/exection/parallel/Validate.kt | 1 - .../exection/parallel/internal/graph/FindCyclesKtTest.kt | 8 +++----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt index b96b235475..8c8cbae2bb 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt @@ -15,7 +15,6 @@ infix operator fun Tasks.invoke( ): Flow = Execution(this, args).invoke() - // ======================= Extensions ======================= /** diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Validate.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Validate.kt index a47694d143..6a5f8a561e 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Validate.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Validate.kt @@ -40,4 +40,3 @@ fun Tasks.validate(initial: ParallelState = emptyMap()): Tasks = run { private fun Iterable>.splitTasks() = this .groupBy { task -> task.type is Parallel.Context } .run { getOrDefault(true, emptyList()) to getOrDefault(false, emptyList()) } - diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt index 9e61db8c3e..f81b49b497 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt @@ -5,7 +5,6 @@ import org.junit.Test class FindCyclesKtTest { - @Test fun valid0() { val graph = mapOf( @@ -26,7 +25,7 @@ class FindCyclesKtTest { val used = mutableSetOf() val graph = nodes.associateWith { used += it - (nodes.shuffled() - used).run { take((0..(size/10)).random()) }.toSet() + (nodes.shuffled() - used).run { take((0..(size / 10)).random()) }.toSet() } // println("=================================") // graph.forEach { println(it) } @@ -62,7 +61,6 @@ class FindCyclesKtTest { assertEquals(expected, actual) } - @Test fun cyclic2() { val graph = mapOf( @@ -121,8 +119,8 @@ class FindCyclesKtTest { 5 to setOf(3) ) val expected = listOf( - listOf(3,1,2,3), - listOf(3,4,5,3), + listOf(3, 1, 2, 3), + listOf(3, 4, 5, 3), ) val actual = graph.findCycles() From 9fafbefa715e1129c24595e270f2ea24448a7e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 09:26:13 +0200 Subject: [PATCH 14/29] Remove wildcards --- .../src/test/kotlin/flank/exection/parallel/ValidateKtTest.kt | 3 ++- .../flank/exection/parallel/internal/graph/FindCyclesKtTest.kt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ValidateKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ValidateKtTest.kt index 1bab556b9b..4e2299bd59 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ValidateKtTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ValidateKtTest.kt @@ -1,6 +1,7 @@ package flank.exection.parallel -import org.junit.Assert.* +import org.junit.Assert.assertEquals +import org.junit.Assert.fail import org.junit.Test import java.lang.NullPointerException diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt index f81b49b497..90b58e5ce2 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/internal/graph/FindCyclesKtTest.kt @@ -1,6 +1,6 @@ package flank.exection.parallel.internal.graph -import org.junit.Assert.* +import org.junit.Assert.assertEquals import org.junit.Test class FindCyclesKtTest { From d7bf5375bfb622cc986e752626a5165c4d833aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 09:34:43 +0200 Subject: [PATCH 15/29] Update doc --- .../parallel/src/main/kotlin/flank/exection/parallel/Execute.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt index 8c8cbae2bb..b1a886679d 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Execute.kt @@ -6,7 +6,6 @@ import kotlinx.coroutines.flow.Flow /** * Execute tasks in parallel with a given args. - * Before executing, the [validate] is performed on a [Tasks] for a given [args] to detect missing dependencies. * * @return [Flow] of [ParallelState] changes. */ From 6d50223acffa1ec7b45709b43f859cc7af219213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 11:38:00 +0200 Subject: [PATCH 16/29] Add example graph --- docs/hld/parallel-example-graph.puml | 62 +++++++++++++++++++ tool/execution/parallel/README.md | 5 ++ .../kotlin/flank/exection/parallel/Reduce.kt | 2 +- 3 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 docs/hld/parallel-example-graph.puml diff --git a/docs/hld/parallel-example-graph.puml b/docs/hld/parallel-example-graph.puml new file mode 100644 index 0000000000..3d2d69f821 --- /dev/null +++ b/docs/hld/parallel-example-graph.puml @@ -0,0 +1,62 @@ +@startuml +'https://plantuml.com/component-diagram +skinparam componentStyle rectangle + +rectangle 7 { +component Finalize +} + +rectangle 6 { +component NotifyRemoteServices +component GenerateReport +} + +rectangle 5 { +component CalculateResults +} + +rectangle 4 { +component PerformExecution +} + +rectangle 3 { +component CollectRemoteData1 +component FetchPreviousResults +} + +rectangle 2 { +component InitialCalculations +component RunRemoteProcess1 +component RunRemoteProcess2 +} + +rectangle 1 { +component CollectLocalData2 +component CollectLocalData1 +component Authorize +} + +Finalize *--> GenerateReport +Finalize *--> NotifyRemoteServices + +GenerateReport *--> CalculateResults +NotifyRemoteServices *--> CalculateResults + +CalculateResults *--> PerformExecution +CalculateResults *--> FetchPreviousResults + +PerformExecution *----> InitialCalculations +PerformExecution *--> CollectRemoteData1 +FetchPreviousResults *---> Authorize + +InitialCalculations *--> CollectLocalData1 +InitialCalculations *--> CollectLocalData2 + +CollectRemoteData1 *--> Authorize +CollectRemoteData1 *--> RunRemoteProcess1 +CollectRemoteData1 *--> RunRemoteProcess2 + +RunRemoteProcess1 *---> Authorize +RunRemoteProcess2 *---> Authorize + +@enduml diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index db74218d4e..7ef913037e 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -15,6 +15,11 @@ Library for task parallelization and asynchronous execution. Imagine a complicated long-running `execution` that needs to perform several operations (`tasks`) in correct order to collect a required `data` and produce `side effects`. Any `execution` like that, can be modeled as set of unrelated data `types` and suspendable functions (`tasks`). +Example + +![example-graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-example-graph.puml) + + ## Type * is unique in the execution scope. diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt index b8d52661e6..3d21212f8e 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt @@ -5,7 +5,7 @@ import flank.exection.parallel.internal.reduceTo /** * Reduce given [Tasks] by [select] types to remove unneeded tasks from the graph. - * The returned graph will only tasks that are returning selected types, their dependencies and derived dependencies. + * The returned graph will hold only tasks that are returning selected types, their dependencies and derived dependencies. * Additionally this is keeping also the validators for initial state. * * @return Reduced [Tasks] From 37bdcf2159a8be8ed918c43af79d60f3ed6bbad4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 11:48:14 +0200 Subject: [PATCH 17/29] Update README.md --- tool/execution/parallel/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index 7ef913037e..cb5f1639a8 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -15,11 +15,10 @@ Library for task parallelization and asynchronous execution. Imagine a complicated long-running `execution` that needs to perform several operations (`tasks`) in correct order to collect a required `data` and produce `side effects`. Any `execution` like that, can be modeled as set of unrelated data `types` and suspendable functions (`tasks`). -Example +#### The example execution graph ![example-graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-example-graph.puml) - ## Type * is unique in the execution scope. From 2fbf09723c8ef3882008cd78514e495b07c22271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 11:52:32 +0200 Subject: [PATCH 18/29] Update README.md --- tool/execution/parallel/README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index cb5f1639a8..cb8b1695a2 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -13,7 +13,9 @@ Library for task parallelization and asynchronous execution. # Design -Imagine a complicated long-running `execution` that needs to perform several operations (`tasks`) in correct order to collect a required `data` and produce `side effects`. Any `execution` like that, can be modeled as set of unrelated data `types` and suspendable functions (`tasks`). +Imagine a complicated long-running `execution` that needs to perform several operations (`tasks`) in correct order to +collect a required `data` and produce `side effects`. Any `execution` like that, can be modeled as set of unrelated +data `types` and suspendable functions (`tasks`). #### The example execution graph @@ -28,7 +30,7 @@ Imagine a complicated long-running `execution` that needs to perform several ope * returned from task. * returned from execution. -### Example +#### Example code ```kotlin object Args : Parallel.Type> @@ -40,7 +42,7 @@ object Logger : Parallel.Type ## Task -* is representing a single atomic operation in the execution process. +* is representing a single atomic operation in the execution process. * is a relation of signature and task function. * `arguments` for task are coming from `initial state` (`arguments`) or results from other `tasks`. * can be uniquely identified using return type of its signature. @@ -67,7 +69,7 @@ object Logger : Parallel.Type ![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/task-graph.puml) -### Example +#### Example code ```kotlin @@ -105,7 +107,7 @@ val execute = setOf( * is used for providing arguments and dependencies for tasks. * is emitted in flow after each change. -### Example +#### Example code ```kotlin // Preparing initial state @@ -117,12 +119,15 @@ val initial = mapOf( ## Execution -* will run tasks in optimized order creating synchronization points where needed and running all other stuff in parallel. +* will run tasks in optimized order creating synchronization points where needed and running all other stuff in + parallel. * is collecting result from each task, and accumulating it to state. * is returning the flow of realtime state changes. * can detach and fail if set of `tasks` are creating broken graph with `missing dependencies`. * Each task in execution scope must have unique return type. This is crucial for producing correct graph. +#### Example code + ```kotlin // For execution without arguments From f6dfad90dea358fbd3081b7c3d972fa0e0f569ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 11:55:50 +0200 Subject: [PATCH 19/29] Update README.md --- tool/execution/parallel/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index cb8b1695a2..fdfc04d4fc 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -17,7 +17,7 @@ Imagine a complicated long-running `execution` that needs to perform several ope collect a required `data` and produce `side effects`. Any `execution` like that, can be modeled as set of unrelated data `types` and suspendable functions (`tasks`). -#### The example execution graph +#### Example execution graph ![example-graph](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-example-graph.puml) @@ -30,7 +30,7 @@ data `types` and suspendable functions (`tasks`). * returned from task. * returned from execution. -#### Example code +#### Example kotlin code ```kotlin object Args : Parallel.Type> @@ -69,7 +69,7 @@ object Logger : Parallel.Type ![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/task-graph.puml) -#### Example code +#### Example kotlin code ```kotlin @@ -107,7 +107,7 @@ val execute = setOf( * is used for providing arguments and dependencies for tasks. * is emitted in flow after each change. -#### Example code +#### Example kotlin code ```kotlin // Preparing initial state @@ -126,7 +126,7 @@ val initial = mapOf( * can detach and fail if set of `tasks` are creating broken graph with `missing dependencies`. * Each task in execution scope must have unique return type. This is crucial for producing correct graph. -#### Example code +#### Example kotlin code ```kotlin From 54e4159273694bad6c9f718db4fd1022ff6cd2f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 12:02:30 +0200 Subject: [PATCH 20/29] Update references --- tool/execution/parallel/README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index fdfc04d4fc..46f67c5971 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -8,8 +8,11 @@ Library for task parallelization and asynchronous execution. * Dependency type - [static](../../docs/architecture.md#static_dependencies) * Public API * Core - [Parallel.kt](./src/main/kotlin/flank/exection/parallel/Parallel.kt) - * Task factories DSL - [Factory.kt](./src/main/kotlin/flank/exection/parallel/Factory.kt) - * Extensions - [Ext.kt](./src/main/kotlin/flank/exection/parallel/Ext.kt) + * Execution - [Execute.kt](./src/main/kotlin/flank/exection/parallel/Execute.kt) + * Task factories DSL - [Create.kt](./src/main/kotlin/flank/exection/parallel/Create.kt) + * Graph validation - [Create.kt](./src/main/kotlin/flank/exection/parallel/Validate.kt) + * Graph reduction - [Reduce.kt](./src/main/kotlin/flank/exection/parallel/Reduce.kt) +* Example usage - [Example.kt](./src/test/kotlin/flank/exection/parallel/Example.kt) # Design @@ -146,9 +149,9 @@ The following diagram is showing parallel execution algorithm in details: ![parallel-execution](http://www.plantuml.com/plantuml/proxy?cache=no&fmt=svg&src=https://raw.githubusercontent.com/Flank/flank/2001_Implement_tool_for_parallel_execution/docs/hld/parallel-execution.puml) -# Relations +# API -The following graphs are showing constrains between elements described previously. +The following graphs are showing constrains between API elements described previously. ### Functions From 169f98c3b03a9222073170de6f87514754d0434e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 13:06:48 +0200 Subject: [PATCH 21/29] Update parallel-execution-api-structures.puml --- .../parallel-execution-api-structures.puml | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/hld/parallel-execution-api-structures.puml b/docs/hld/parallel-execution-api-structures.puml index 0f0fa7aa42..2ff63e5814 100644 --- a/docs/hld/parallel-execution-api-structures.puml +++ b/docs/hld/parallel-execution-api-structures.puml @@ -10,13 +10,14 @@ class "Parallel.Task" as Task { signature: Signature execute: ExecuteTask } -class "Parallel.Task.Signature" as Task_Signature{ +class "Parallel.Task.Signature" as Signature{ returns: Type args: Set> } class "Parallel.Function" as Function { = suspend (X.() -> R) -> ExecuteTask } +interface "Any data" as Data << (R, yellow) >> interface ExecuteTask << (T, orchid) >> { = suspend ParallelState.() -> R } @@ -28,15 +29,28 @@ interface Output << (T, orchid) >> { = Any.() -> Unit } -Output "0..1" ..o "1" Context + +Function ..> ExecuteTask + ParallelState "1" --o "1" Context -Type <|-- Context +ParallelState "1" o- "*" Type + +ExecuteTask "1" -* "1" Task +ExecuteTask o.. ParallelState +ExecuteTask ..> Data + +'Data "*" -o "1" ParallelState +'Data "*" -o "1" ParallelState +ParallelState "1" o- "*" Data +Data .. Type + + +Task "1" *--- "1" Signature -Type "1" --* "1" Task_Signature -Type "*" --o "1" Task_Signature -Task_Signature "1" --* "*" Task +Type "1" -* "1" Signature +Type "*" -o "1" Signature +Context -|> Type -Function "1" ..> "*" ExecuteTask -ExecuteTask "1" --* "1" Task +Context "0..1" o..> "1" Output @enduml From 76e1225b6f2b7831da8d02afedda9902175eca00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 13:32:19 +0200 Subject: [PATCH 22/29] Update parallel-execution-api-structures.puml --- .../parallel-execution-api-structures.puml | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/hld/parallel-execution-api-structures.puml b/docs/hld/parallel-execution-api-structures.puml index 2ff63e5814..2095628f49 100644 --- a/docs/hld/parallel-execution-api-structures.puml +++ b/docs/hld/parallel-execution-api-structures.puml @@ -1,8 +1,5 @@ @startuml - - package "parallel" { - class "Parallel.Context" as Context { out: Output? } @@ -15,12 +12,16 @@ returns: Type args: Set> } class "Parallel.Function" as Function { -= suspend (X.() -> R) -> ExecuteTask +context: () -> X += suspend (ExecuteTaskInContext) -> ExecuteTask } interface "Any data" as Data << (R, yellow) >> interface ExecuteTask << (T, orchid) >> { = suspend ParallelState.() -> R } +interface ExecuteTaskInContext << (T, orchid) >> { += suspend X.() -> R +} interface "Parallel.Type" as Type interface ParallelState << (T, orchid) >> { = Map, Any> @@ -29,28 +30,27 @@ interface Output << (T, orchid) >> { = Any.() -> Unit } - Function ..> ExecuteTask +Function #.. ExecuteTaskInContext -ParallelState "1" --o "1" Context -ParallelState "1" o- "*" Type +ExecuteTaskInContext #.. Context +ExecuteTaskInContext .> Data + +Data <. ExecuteTask +Data "*" --o "1" ParallelState ExecuteTask "1" -* "1" Task -ExecuteTask o.. ParallelState -ExecuteTask ..> Data +ExecuteTask #.. ParallelState -'Data "*" -o "1" ParallelState -'Data "*" -o "1" ParallelState -ParallelState "1" o- "*" Data -Data .. Type +Context --|> Type +Context "0..1" o..> "1" Output +ParallelState "1" -o "1" Context +ParallelState "1" o- "*" Type -Task "1" *--- "1" Signature +Task "1" *-- "1" Signature Type "1" -* "1" Signature Type "*" -o "1" Signature -Context -|> Type - -Context "0..1" o..> "1" Output @enduml From 72245e8e2c44ab65cf73b0dddb5f8b392cdbc073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 13:51:44 +0200 Subject: [PATCH 23/29] Fix flaky test --- .../src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt index a849852ca4..0caf26dbef 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt @@ -125,8 +125,8 @@ class ExecuteKtTest { @Test fun `abort execution`() { val execute = setOf( - A using { delay(50); throw Exception() }, - B using { delay(100) }, + A using { delay(100); throw Exception() }, + B using { delay(300) }, C from setOf(B) using { } ) val actual = runBlocking { execute().last() } From 22143126d1675ee36c463bb982f8c37e3cd8da84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 14:01:02 +0200 Subject: [PATCH 24/29] Update parallel-execution-api-functions.puml --- docs/hld/parallel-execution-api-functions.puml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/hld/parallel-execution-api-functions.puml b/docs/hld/parallel-execution-api-functions.puml index a5cd57e482..a0b0267d8c 100644 --- a/docs/hld/parallel-execution-api-functions.puml +++ b/docs/hld/parallel-execution-api-functions.puml @@ -10,8 +10,8 @@ class "Parallel.Task" as Task class "Parallel.Task.Signature" as Task_Signature interface "Parallel.Type" as Type interface "Parallel.Type" as Type2 -interface ExecuteTask -interface ParallelState +interface ExecuteTask << (T, orchid) >> +interface ParallelState << (T, orchid) >> object "using" as usingType { creates @@ -30,7 +30,7 @@ object "invoke()" as reduce { reduces } -Function "1" ..> ExecuteTask +ExecuteTask <. "1" Function ExecuteTask <--o "1" usingSignature usingSignature "1" #--> Task_Signature @@ -51,8 +51,9 @@ Task <. "*" reduce Task <-# "*" reduce Task <--# "*" invokeExecution -invokeExecution "1" o--> ParallelState -invokeExecution "*" ..> ParallelState + +invokeExecution "*" .left.> ParallelState +invokeExecution "1" o-left-> ParallelState } @enduml From 5bd3d32e9bd563808b26a5696ade4a5861579b35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Fri, 11 Jun 2021 14:30:02 +0200 Subject: [PATCH 25/29] Small fix --- .../main/kotlin/flank/exection/parallel/internal/Reduce.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt index 083dc6745f..ae1d1ce5c2 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt @@ -31,12 +31,12 @@ private tailrec fun Tasks.reduce( when { isEmpty() -> acc else -> flatMap(Parallel.Task<*>::args) - .mapNotNull(all::findByReturnType) + .mapNotNull(all::findByType) .toSet() .reduce(all, acc + this) } -private fun Tasks.findByReturnType( +private fun Tasks.findByType( type: Parallel.Type<*> ): Parallel.Task<*>? = find { task -> task.type == type } From 4926c2596832f6a498437c5431105c42191f1587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Sun, 13 Jun 2021 17:51:45 +0200 Subject: [PATCH 26/29] Remove unneeded values from state when expected types are specified. Optimize reduce graph algorithm. Add benchmarks for core functions Small fixes --- .../flank/exection/parallel/Parallel.kt | 10 ++- .../kotlin/flank/exection/parallel/Reduce.kt | 12 +-- .../parallel/internal/ContextProvider.kt | 8 +- .../exection/parallel/internal/Execution.kt | 32 +++++++- .../exection/parallel/internal/Prepare.kt | 4 +- .../exection/parallel/internal/Property.kt | 7 +- .../exection/parallel/internal/Reduce.kt | 52 ++++-------- .../parallel/internal/graph/FindCycles.kt | 19 +++-- .../internal/graph/FindDependencies.kt | 22 ++++++ .../kotlin/flank/exection/parallel/Example.kt | 79 +++++++++++-------- .../flank/exection/parallel/ExecuteKtTest.kt | 34 ++++++++ .../exection/parallel/benchmark/Execute.kt | 38 +++++++++ .../exection/parallel/benchmark/Reduce.kt | 37 +++++++++ .../exection/parallel/benchmark/Validate.kt | 35 ++++++++ 14 files changed, 293 insertions(+), 96 deletions(-) create mode 100644 tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindDependencies.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Execute.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Reduce.kt create mode 100644 tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Validate.kt diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt index 625306397c..e92c021da4 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Parallel.kt @@ -61,7 +61,8 @@ object Parallel { */ data class Task( val signature: Signature, - val execute: ExecuteTask + val execute: ExecuteTask, + val expected: Boolean = true, ) { /** * The task signature. @@ -78,7 +79,7 @@ object Parallel { /** * Parameterized factory for creating task functions in scope of [X]. */ - class Function(override val context: () -> X) : ContextProvider + class Function(override val context: () -> X) : ContextProvider() data class Event internal constructor( val type: Type<*>, @@ -115,6 +116,11 @@ typealias Output = Any.() -> Unit */ typealias ParallelState = Map, Any> +/** + * Immutable state for parallel execution. + */ +typealias Property = Pair, Any> + /** * Type for group of parallel tasks. Each task must be unique in group. */ diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt index 3d21212f8e..f7fc6bf563 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/Reduce.kt @@ -1,23 +1,23 @@ package flank.exection.parallel -import flank.exection.parallel.internal.initialValidators +import flank.exection.parallel.internal.contextValidators import flank.exection.parallel.internal.reduceTo /** - * Reduce given [Tasks] by [select] types to remove unneeded tasks from the graph. + * Reduce given [Tasks] by [expected] types to remove unneeded tasks from the graph. * The returned graph will hold only tasks that are returning selected types, their dependencies and derived dependencies. * Additionally this is keeping also the validators for initial state. * * @return Reduced [Tasks] */ operator fun Tasks.invoke( - select: Set> + expected: Set> ): Tasks = - reduceTo(select + initialValidators) + reduceTo(expected + contextValidators()) /** * Shortcut for tasks reducing. */ operator fun Tasks.invoke( - vararg returns: Parallel.Type<*> -): Tasks = invoke(returns.toSet()) + vararg expected: Parallel.Type<*> +): Tasks = invoke(expected.toSet()) diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt index 587ed58032..105f3fdcc7 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/ContextProvider.kt @@ -2,12 +2,16 @@ package flank.exection.parallel.internal import flank.exection.parallel.ExecuteTask import flank.exection.parallel.Parallel +import flank.exection.parallel.validator /** * Abstract factory for creating task function. */ -internal interface ContextProvider { - val context: () -> X +abstract class ContextProvider { + protected abstract val context: () -> X + operator fun invoke(body: suspend X.() -> R): ExecuteTask = { context().also { it.state = this }.body() } + + val validator by lazy { validator(context) } } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt index ee70ab04e5..26c5788ceb 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Execution.kt @@ -6,6 +6,7 @@ import flank.exection.parallel.Parallel.Logger import flank.exection.parallel.Parallel.Task import flank.exection.parallel.Parallel.Type import flank.exection.parallel.ParallelState +import flank.exection.parallel.Property import flank.exection.parallel.Tasks import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -19,6 +20,7 @@ import kotlinx.coroutines.flow.onCompletion import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.scan import kotlinx.coroutines.launch +import java.util.concurrent.atomic.AtomicInteger /** * Invoke the given [Execution] in parallel. @@ -30,7 +32,7 @@ internal operator fun Execution.invoke(): Flow = .onEach { (type, value) -> if (value is Throwable && isNotClosed()) abortBy(type, value) } // Accumulate each received value in state. - .scan(initial) { state, value -> state + value } + .scan(initial, updateState()) // Handle state changes. .onEach { state: ParallelState -> @@ -90,13 +92,21 @@ internal class Execution( /** * The set of value types required to complete the execution. */ - val required = tasks.map(Task<*>::type).toSet() + val required = tasks.filter(Task<*>::expected).map(Task<*>::type).toSet() /** * Map of remaining tasks for run grouped by arguments. */ val remaining = tasks.groupBy(Task<*>::args).toMutableMap() + /** + * Reference counter for state types marked as not expected. + * Values of types that are not expected but required as dependencies, + * can be removed when there are no remaining tasks depending on them. + */ + val references = tasks.flatMap(Task<*>::args).groupBy { it } + .minus(required).mapValues { (_, refs) -> AtomicInteger(refs.size) } + /** * Reference to optional output for structural logs. */ @@ -130,6 +140,17 @@ private suspend fun Execution.abortBy(type: Type<*>, cause: Throwable) { channel.close() } +/** + * Create function for updating state depending on reference counter state. + */ +private fun Execution.updateState(): suspend (ParallelState, Property) -> ParallelState = + if (references.isEmpty()) ParallelState::plus + else { state, property -> + references.filterValues { counter -> counter.compareAndSet(0, 0) } + .map { (type, _) -> type } + .let { junks -> state + property - junks } + } + /** * The execution is complete when all required types was accumulated to state. */ @@ -157,7 +178,7 @@ private fun Execution.filterTasksFor(state: ParallelState): Map>, Li channel.isEmpty.not() || // some values are waiting in the channel queue. throw DeadlockError(state, jobs, remaining) - // Remove from queue the tasks for current iteration. + // Remove from queue tasks for current iteration. remaining -= keys } @@ -185,6 +206,11 @@ private fun Execution.execute( task: Task<*>, args: Map, Any>, ) { + // Decrement references to arguments types + if (references.isNotEmpty()) task.args.forEach { type -> + references[type]?.getAndUpdate { count -> count - 1 } + } + // Obtain type of current task. val type: Type<*> = task.type diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt index 4aab7b76f7..5ab8b3d2a6 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Prepare.kt @@ -7,5 +7,5 @@ import flank.exection.parallel.Tasks * Get initial state validators. * This is necessary to perform validations of initial state before the execution. */ -internal val Tasks.initialValidators: List - get() = mapNotNull { task -> task.type as? Parallel.Context } +internal fun Tasks.contextValidators(): List = + mapNotNull { task -> task.type as? Parallel.Context } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Property.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Property.kt index 615fb00bec..e39821366f 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Property.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Property.kt @@ -7,6 +7,7 @@ import flank.exection.parallel.ParallelState * Factory function for lazy property delegate. */ internal fun Parallel.Context.lazyProperty(type: Parallel.Type) = lazy { + @Suppress("UNCHECKED_CAST") state[type] as T } @@ -16,12 +17,12 @@ internal fun Parallel.Context.lazyProperty(type: Parallel.Type) = l class EagerProperties( private val state: () -> ParallelState ) { - private val set = mutableSetOf>() + private val props = mutableSetOf>() /** * Initialize eager properties, this performs also validation. */ - operator fun invoke(): Unit = set.forEach { prop -> println(prop.value) } + operator fun invoke(): Unit = props.forEach { prop -> prop.value } /** * Register new parallel type. Inline modifier is necessary to perform real type check @@ -29,6 +30,6 @@ class EagerProperties( inline operator fun invoke(type: Parallel.Type): Lazy = lazy { type.value() as T }.append() // local helpers need to be public because of inlined invoke - fun Lazy.append(): Lazy = apply { set.plusAssign(this) } + fun Lazy.append(): Lazy = apply { props += this } fun Parallel.Type.value(): Any? = state()[this] } diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt index ae1d1ce5c2..0f0aaae4c9 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/Reduce.kt @@ -1,42 +1,22 @@ package flank.exection.parallel.internal -import flank.exection.parallel.Parallel +import flank.exection.parallel.Parallel.Task +import flank.exection.parallel.Parallel.Type import flank.exection.parallel.Tasks +import flank.exection.parallel.internal.graph.findDependenciesIn internal infix fun Tasks.reduceTo( - selectedTypes: Set> -): Tasks = - filter { task -> task.type in selectedTypes } - .toSet() - .apply { - val notFound = selectedTypes - map(Parallel.Task<*>::type) - if (notFound.isNotEmpty()) throw Exception("Cannot reduce find tasks for the following types: $notFound") + expected: Set> +): Tasks { + val notFound = expected - map(Task<*>::type) + if (notFound.isNotEmpty()) throw Exception("Cannot find tasks for the following types: $notFound") + val graph: Map, Set>> = associate { task -> task.type to task.args } + val dependencies = expected.findDependenciesIn(graph) + return mapNotNull { task -> + when (task.type) { + in expected -> task + in dependencies -> task.copy(expected = false) + else -> null } - .reduce(this) - -/** - * Reduce [all] steps to given receiver steps and their dependencies. - * - * @receiver The task selector for current reducing step. - * @param all The list of all tasks that are under reducing. - * @param acc Currently accumulated tasks. - * @return Accumulated tasks if selector is empty. - */ -private tailrec fun Tasks.reduce( - all: Tasks, - acc: Tasks = - if (isEmpty()) all - else emptySet(), -): Tasks = - when { - isEmpty() -> acc - else -> flatMap(Parallel.Task<*>::args) - .mapNotNull(all::findByType) - .toSet() - .reduce(all, acc + this) - } - -private fun Tasks.findByType( - type: Parallel.Type<*> -): Parallel.Task<*>? = - find { task -> task.type == type } + }.toSet() +} diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindCycles.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindCycles.kt index 73d079535c..5cfac23416 100644 --- a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindCycles.kt +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindCycles.kt @@ -1,11 +1,17 @@ package flank.exection.parallel.internal.graph -tailrec fun Map>.findCycles( - - remaining: List = toList() - .sortedByDescending { (_, edges) -> edges.size } - .map { (vertices, _) -> vertices } - .toMutableList(), +/** + * Find cycles in given graph. + * + * @receiver Graph to search. + * @return List of cycles. Each cycle is a list of nodes. + */ +internal tailrec fun Map>.findCycles( + + remaining: Set = toList() + .sortedByDescending { (_, children) -> children.size } + .map { (parent, _) -> parent } + .toSet(), queue: List = emptyList(), @@ -28,7 +34,6 @@ tailrec fun Map>.findCycles( val cycle = current in path + next // println("$cycle R:$remaining Q:$queue N:$next C:$current P:$path V:$visited") - // println() return findCycles( remaining = diff --git a/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindDependencies.kt b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindDependencies.kt new file mode 100644 index 0000000000..fbb288a093 --- /dev/null +++ b/tool/execution/parallel/src/main/kotlin/flank/exection/parallel/internal/graph/FindDependencies.kt @@ -0,0 +1,22 @@ +package flank.exection.parallel.internal.graph + +/** + * Find dependencies for given nodes in [graph]. + * + * @receiver Expected elements for current iteration. + * @param graph Graph to search. + * @param acc Currently accumulated elements. + * @return Set expected elements along with dependencies. + */ +internal tailrec fun Set.findDependenciesIn( + graph: Map>, + acc: Set = + if (isEmpty()) graph.keys + else emptySet(), +): Set = + when { + isEmpty() -> acc // No more elements, so return all accumulated. + else -> flatMap(graph::getValue).toSet() // Map each expected element to its dependencies. + .minus(acc) // Remove already accumulated elements to optimize calculations. + .findDependenciesIn(graph, acc + this) // Accumulate current dependencies and run next iteration. + } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt index 98bd90fd55..912e674268 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/Example.kt @@ -6,7 +6,7 @@ import flank.exection.parallel.Example.Summary import flank.exection.parallel.Parallel.Type import kotlinx.coroutines.delay import kotlinx.coroutines.flow.asFlow -import kotlinx.coroutines.flow.last +import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking @@ -14,22 +14,6 @@ import kotlinx.coroutines.runBlocking // ======================= Public API ======================= private object Example { - - /** - * Context for [Example] scope. - * Wrapper for state values with exposed static accessors. - */ - class Context : Parallel.Context() { - // Initial part of state. - // Validated before execution - val args by !Args - // Values evaluated during execution by tasks. - val a by -A - val b by -B - val c by -C - val hello by -Hello - } - /** * Arguments for [Example]. */ @@ -53,6 +37,27 @@ private object Example { object Failing : Type } + /** + * Context for [Example] scope. + * Wrapper for state values with exposed static accessors. + */ + class Context : Parallel.Context() { + // Initial part of state. + // Validated before execution + val args by !Args + + // Values evaluated during execution by tasks. + val a by -A + val b by -B + val c by -C + val hello by -Hello + } + + /** + * Factory method for creating task functions with [Context]. + */ + val context = Parallel.Function(::Context) + /** * List of tasks in [Example] scope */ @@ -68,11 +73,6 @@ private object Example { ) } - /** - * Factory method for creating task functions with [Context]. - */ - private val context = Parallel.Function(::Context) - // ======================= Tasks ======================= private val validate: Parallel.Task = validator(::Context) @@ -118,6 +118,27 @@ private object Example { // ======================= Example Run ======================= fun main() { + val args = mapOf( + // Specify initial state. + // Commend initial Args to simulate failure of initial validation. + Args to Args( + wait = 50, + a = 5, + b = 8, + c = 13, + ), + Parallel.Logger to { log: Any -> + // println(log) + }, + ) + + Example.run { + execute + context.validator + }.validate( + // Comment line below to simulate error on context.validator + args + ) + runBlocking { Example.execute( // Expect selected tasks. @@ -125,19 +146,7 @@ fun main() { Hello, // Uncomment to simulate failing execution. // Hello.Failing, - )( - // Specify initial state. - // Commend initial Args to simulate failure of initial validation. - Args to Args( - wait = 50, - a = 5, - b = 8, - c = 13, - ), - Parallel.Logger to { log: Any -> - println(log) - }, - ).last().let { state -> + )(args).collect { state -> // Print final state println() state.forEach(::println) diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt index 0caf26dbef..bd229b0b56 100644 --- a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/ExecuteKtTest.kt @@ -4,6 +4,8 @@ import kotlinx.coroutines.CancellationException import kotlinx.coroutines.delay import kotlinx.coroutines.flow.collect import kotlinx.coroutines.flow.last +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.toList import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withTimeout @@ -20,6 +22,7 @@ object B : Parallel.Type object C : Parallel.Type object D : Parallel.Type object E : Parallel.Type +object F : Parallel.Type // ====================== TESTS ====================== @@ -162,4 +165,35 @@ class ExecuteKtTest { assert(actual[A] is NullPointerException) } + + /** + * Values of types that are not expected but required as dependencies, + * can be automatically removed when there are no remaining tasks depending on them. + */ + @Test + fun `removing unneeded values`() { + val execute = setOf( + A from setOf(B) using {}, + B from setOf(E, C) using {}, + C from setOf(F, E, D) using {}, + D from setOf(E, F) using {}, + E from setOf(F) using {}, + F using {}, + ) + val expected = listOf( + setOf(F), + setOf(F, E), + setOf(F, E, D), + setOf(E, C), + setOf(B), + setOf(A), + ) + val actual = runBlocking { + execute(A)() + .onEach { state -> println(); state.forEach(::println) } + .map { state -> state.keys } + .toList() + } + assertEquals(expected, actual) + } } diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Execute.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Execute.kt new file mode 100644 index 0000000000..1db61c662c --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Execute.kt @@ -0,0 +1,38 @@ +package flank.exection.parallel.benchmark + +import flank.exection.parallel.Parallel +import flank.exection.parallel.Tasks +import flank.exection.parallel.from +import flank.exection.parallel.internal.args +import flank.exection.parallel.invoke +import flank.exection.parallel.using +import kotlinx.coroutines.runBlocking +import kotlin.system.measureTimeMillis + +fun main() { + data class Type(val id: Int) : Parallel.Type + + val types = (0..1000).map { Type(it) } + + val used = mutableSetOf>() + + val execute: Tasks = types.map { returns -> + used += returns + val args = (types - used) + .shuffled() + .run { take((0..size / 10).random()) } + .toSet() + returns from args using {} + }.toSet() + + val edges = execute.sumOf { task -> task.args.size } + + println("edges: $edges") + println() + + runBlocking { + repeat(100) { + measureTimeMillis { execute.invoke() }.let(::println) + } + } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Reduce.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Reduce.kt new file mode 100644 index 0000000000..05dcf5b80a --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Reduce.kt @@ -0,0 +1,37 @@ +package flank.exection.parallel.benchmark + +import flank.exection.parallel.Parallel +import flank.exection.parallel.Tasks +import flank.exection.parallel.from +import flank.exection.parallel.internal.args +import flank.exection.parallel.invoke +import flank.exection.parallel.using +import kotlin.system.measureTimeMillis + +fun main() { + data class Type(val id: Int) : Parallel.Type + + val types = (0..1000).map { Type(it) } + + val used = mutableSetOf>() + + val execute: Tasks = types.map { returns -> + used += returns + val args = (types - used) + .shuffled() + .run { take((0..size / 10).random()) } + .toSet() + returns from args using {} + }.toSet() + + val expected = types.take(1).toSet() + + val edges = execute.sumOf { task -> task.args.size } + + println("edges: $edges") + println() + + repeat(100) { + measureTimeMillis { execute.invoke(expected) }.let(::println) + } +} diff --git a/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Validate.kt b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Validate.kt new file mode 100644 index 0000000000..68d27a92f8 --- /dev/null +++ b/tool/execution/parallel/src/test/kotlin/flank/exection/parallel/benchmark/Validate.kt @@ -0,0 +1,35 @@ +package flank.exection.parallel.benchmark + +import flank.exection.parallel.Parallel +import flank.exection.parallel.Tasks +import flank.exection.parallel.from +import flank.exection.parallel.internal.args +import flank.exection.parallel.using +import flank.exection.parallel.validate +import kotlin.system.measureTimeMillis + +fun main() { + data class Type(val id: Int) : Parallel.Type + + val types = (0..1000).map { Type(it) } + + val used = mutableSetOf>() + + val execute: Tasks = types.map { returns -> + used += returns + val args = (types - used) + .shuffled() + .run { take((0..size / 10).random()) } + .toSet() + returns from args using {} + }.toSet() + + val edges = execute.sumOf { task -> task.args.size } + + println("edges: $edges") + println() + + repeat(10) { + measureTimeMillis { execute.validate() }.let(::println) + } +} From cbb0a0b53294338fca1dbdfd7a9b261a93c803b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 14 Jun 2021 11:53:33 +0200 Subject: [PATCH 27/29] Update diagram --- docs/hld/parallel-example-graph.puml | 32 ++++++++++------------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/docs/hld/parallel-example-graph.puml b/docs/hld/parallel-example-graph.puml index 3d2d69f821..9727979f1c 100644 --- a/docs/hld/parallel-example-graph.puml +++ b/docs/hld/parallel-example-graph.puml @@ -2,39 +2,25 @@ 'https://plantuml.com/component-diagram skinparam componentStyle rectangle -rectangle 7 { component Finalize -} -rectangle 6 { component NotifyRemoteServices component GenerateReport -} -rectangle 5 { component CalculateResults -} -rectangle 4 { component PerformExecution -} -rectangle 3 { component CollectRemoteData1 component FetchPreviousResults -} -rectangle 2 { component InitialCalculations component RunRemoteProcess1 component RunRemoteProcess2 -} -rectangle 1 { component CollectLocalData2 component CollectLocalData1 component Authorize -} Finalize *--> GenerateReport Finalize *--> NotifyRemoteServices @@ -45,18 +31,22 @@ NotifyRemoteServices *--> CalculateResults CalculateResults *--> PerformExecution CalculateResults *--> FetchPreviousResults -PerformExecution *----> InitialCalculations -PerformExecution *--> CollectRemoteData1 -FetchPreviousResults *---> Authorize - InitialCalculations *--> CollectLocalData1 InitialCalculations *--> CollectLocalData2 -CollectRemoteData1 *--> Authorize +PerformExecution *--> CollectRemoteData1 +PerformExecution *---> InitialCalculations + +FetchPreviousResults *--> Authorize + +InitialCalculations -[hidden]r- RunRemoteProcess1 + CollectRemoteData1 *--> RunRemoteProcess1 CollectRemoteData1 *--> RunRemoteProcess2 -RunRemoteProcess1 *---> Authorize -RunRemoteProcess2 *---> Authorize +RunRemoteProcess1 *--> Authorize +RunRemoteProcess2 *--> Authorize +RunRemoteProcess2 -[hidden]r- FetchPreviousResults + @enduml From 1838aa779a29883d9386b62019db6e3a449d0f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janek=20G=C3=B3ral?= Date: Mon, 14 Jun 2021 13:24:02 +0200 Subject: [PATCH 28/29] Fix gradle version --- flank-scripts/gradle/wrapper/gradle-wrapper.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- test_runner/gradle/wrapper/gradle-wrapper.properties | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/flank-scripts/gradle/wrapper/gradle-wrapper.properties b/flank-scripts/gradle/wrapper/gradle-wrapper.properties index 7fb030ba10..e85b8e3289 100644 --- a/flank-scripts/gradle/wrapper/gradle-wrapper.properties +++ b/flank-scripts/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 7fb030ba10..e85b8e3289 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/test_runner/gradle/wrapper/gradle-wrapper.properties b/test_runner/gradle/wrapper/gradle-wrapper.properties index 3ff35d4a25..c047c05416 100644 --- a/test_runner/gradle/wrapper/gradle-wrapper.properties +++ b/test_runner/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-1-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.1-rc-2-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists From 7a6bcda9c86b2f47f8853efcbc4dd431beb6042e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20G=C3=B3ral?= <60390247+jan-gogo@users.noreply.github.com> Date: Wed, 16 Jun 2021 11:19:31 +0200 Subject: [PATCH 29/29] Apply suggestions from code review Co-authored-by: Michael Wright --- tool/execution/parallel/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tool/execution/parallel/README.md b/tool/execution/parallel/README.md index 46f67c5971..2661ddbd52 100644 --- a/tool/execution/parallel/README.md +++ b/tool/execution/parallel/README.md @@ -17,7 +17,7 @@ Library for task parallelization and asynchronous execution. # Design Imagine a complicated long-running `execution` that needs to perform several operations (`tasks`) in correct order to -collect a required `data` and produce `side effects`. Any `execution` like that, can be modeled as set of unrelated +collect required `data` and produce `side effects`. Any `execution` like that, can be modeled as a set of unrelated data `types` and suspendable functions (`tasks`). #### Example execution graph @@ -48,11 +48,11 @@ object Logger : Parallel.Type * is representing a single atomic operation in the execution process. * is a relation of signature and task function. * `arguments` for task are coming from `initial state` (`arguments`) or results from other `tasks`. -* can be uniquely identified using return type of its signature. +* can be uniquely identified using the return type of its signature. ### Signature -* is a set of argument `types` related with one return `type` just like in normal functions. +* is a set of argument `types` related with one return `type` just like in a normal functions. ### Function (ExecuteTask)