851: Refactor data loaders and disable caching

backend/src/main/kotlin/app/ehrenamtskarte/backend/application/webservice/applicationGraphQLParams.kt

@@ -1,11 +1,10 @@
 package app.ehrenamtskarte.backend.application.webservice
 
-import app.ehrenamtskarte.backend.application.webservice.dataloader.APPLICATION_LOADER_NAME
-import app.ehrenamtskarte.backend.application.webservice.dataloader.VERIFICATIONS_BY_APPLICATION_LOADER_NAME
 import app.ehrenamtskarte.backend.application.webservice.dataloader.applicationLoader
 import app.ehrenamtskarte.backend.application.webservice.dataloader.verificationsByApplicationLoader
 import app.ehrenamtskarte.backend.application.webservice.schema.create.primitives.UploadKey
 import app.ehrenamtskarte.backend.common.webservice.GraphQLParams
+import app.ehrenamtskarte.backend.common.webservice.createRegistryFromNamedDataLoaders
 import com.expediagroup.graphql.generator.SchemaGeneratorConfig
 import com.expediagroup.graphql.generator.TopLevelObject
 import com.expediagroup.graphql.generator.hooks.SchemaGeneratorHooks
@@ -16,16 +15,8 @@ import graphql.schema.CoercingParseValueException
 import graphql.schema.CoercingSerializeException
 import graphql.schema.GraphQLScalarType
 import graphql.schema.GraphQLType
-import org.dataloader.DataLoaderRegistry
 import kotlin.reflect.KType
 
-private fun createDataLoaderRegistry(): DataLoaderRegistry {
-    val dataLoaderRegistry = DataLoaderRegistry()
-    dataLoaderRegistry.register(APPLICATION_LOADER_NAME, applicationLoader)
-    dataLoaderRegistry.register(VERIFICATIONS_BY_APPLICATION_LOADER_NAME, verificationsByApplicationLoader)
-    return dataLoaderRegistry
-}
-
 val Upload: GraphQLScalarType = GraphQLScalarType.newScalar()
     .name("Upload")
     .description("A file part in a multipart request")
@@ -70,7 +61,10 @@ val applicationGraphQlParams = GraphQLParams(
                 }
         },
     ),
-    dataLoaderRegistry = createDataLoaderRegistry(),
+    dataLoaderRegistry = createRegistryFromNamedDataLoaders(
+        applicationLoader,
+        verificationsByApplicationLoader,
+    ),
     queries = listOf(
         TopLevelObject(EakApplicationQueryService()),
     ),

backend/src/main/kotlin/app/ehrenamtskarte/backend/application/webservice/dataloader/ApplicationDataLoader.kt

@@ -7,58 +7,45 @@ import app.ehrenamtskarte.backend.application.database.Applications
 import app.ehrenamtskarte.backend.application.database.repos.ApplicationRepository
 import app.ehrenamtskarte.backend.application.webservice.schema.view.ApplicationVerificationView
 import app.ehrenamtskarte.backend.application.webservice.schema.view.ApplicationView
-import kotlinx.coroutines.runBlocking
-import org.dataloader.DataLoader
-import org.dataloader.DataLoaderFactory
+import app.ehrenamtskarte.backend.common.webservice.newNamedDataLoader
 import org.jetbrains.exposed.sql.SortOrder
 import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
-import java.util.concurrent.CompletableFuture
 
-const val APPLICATION_LOADER_NAME = "APPLICATION_LOADER"
-
-val applicationLoader: DataLoader<Int, ApplicationView?> = DataLoaderFactory.newDataLoader { ids ->
-    CompletableFuture.supplyAsync {
-        runBlocking {
-            transaction {
-                ApplicationRepository.findByIds(ids).map { it?.let { ApplicationView.fromDbEntity(it) } }
-            }
+val applicationLoader = newNamedDataLoader("APPLICATION_LOADER") { ids ->
+    transaction {
+        ApplicationRepository.findByIds(ids).map {
+            it?.let { ApplicationView.fromDbEntity(it) }
         }
     }
 }
 
-const val VERIFICATIONS_BY_APPLICATION_LOADER_NAME = "VERIFICATIONS_BY_APPLICATION_LOADER"
-val verificationsByApplicationLoader: DataLoader<Int, List<ApplicationVerificationView>?> =
-    DataLoaderFactory.newDataLoader { ids ->
-        CompletableFuture.supplyAsync {
-            runBlocking {
-                transaction {
-                    val list = (Applications leftJoin ApplicationVerifications)
-                        .slice(listOf(Applications.id).plus(ApplicationVerifications.columns))
-                        .select { Applications.id inList ids }
-                        .orderBy(Applications.id to SortOrder.ASC, ApplicationVerifications.id to SortOrder.ASC)
-                        .toList()
-                    val groupedByApplication = list.groupBy { row -> row[Applications.id].value }
-                    val entities = ids.map { id ->
-                        groupedByApplication[id]?.let { list ->
-                            val verificationEntities = list.mapNotNull {
-                                if (it[ApplicationVerifications.id.nullable()] == null) {
-                                    null
-                                } else {
-                                    ApplicationVerificationEntity.wrapRow(it)
-                                }
-                            }
-                            verificationEntities
-                        }
-                    }
-                    entities.map {
-                        it?.let { verificationEntities ->
-                            verificationEntities.map { entity ->
-                                ApplicationVerificationView.fromDbEntity(entity)
-                            }
-                        }
+val verificationsByApplicationLoader = newNamedDataLoader<Int, _>("VERIFICATIONS_BY_APPLICATION_LOADER") { ids ->
+    transaction {
+        val list = (Applications leftJoin ApplicationVerifications)
+            .slice(listOf(Applications.id).plus(ApplicationVerifications.columns))
+            .select { Applications.id inList ids }
+            .orderBy(Applications.id to SortOrder.ASC, ApplicationVerifications.id to SortOrder.ASC)
+            .toList()
+        val groupedByApplication = list.groupBy { row -> row[Applications.id].value }
+        val entities = ids.map { id ->
+            groupedByApplication[id]?.let { list ->
+                val verificationEntities = list.mapNotNull {
+                    if (it[ApplicationVerifications.id.nullable()] == null) {
+                        null
+                    } else {
+                        ApplicationVerificationEntity.wrapRow(it)
                     }
                 }
+                verificationEntities
+            }
+        }
+        entities.map {
+            it?.let { verificationEntities ->
+                verificationEntities.map { entity ->
+                    ApplicationVerificationView.fromDbEntity(entity)
+                }
             }
         }
     }
+}

backend/src/main/kotlin/app/ehrenamtskarte/backend/application/webservice/schema/view/ApplicationView.kt

@@ -1,7 +1,8 @@
 package app.ehrenamtskarte.backend.application.webservice.schema.view
 
 import app.ehrenamtskarte.backend.application.database.ApplicationEntity
-import app.ehrenamtskarte.backend.application.webservice.dataloader.VERIFICATIONS_BY_APPLICATION_LOADER_NAME
+import app.ehrenamtskarte.backend.application.webservice.dataloader.verificationsByApplicationLoader
+import app.ehrenamtskarte.backend.common.webservice.fromEnvironment
 import graphql.schema.DataFetchingEnvironment
 import java.util.concurrent.CompletableFuture
 
@@ -12,9 +13,6 @@ data class ApplicationView(val id: Int, val regionId: Int, val createdDate: Stri
             ApplicationView(entity.id.value, entity.regionId.value, entity.createdDate.toString(), entity.jsonValue)
     }
 
-    fun verifications(dfe: DataFetchingEnvironment): CompletableFuture<List<ApplicationVerificationView>> {
-        return dfe.getDataLoader<Int, List<ApplicationVerificationView>?>(
-            VERIFICATIONS_BY_APPLICATION_LOADER_NAME,
-        ).load(this.id)!!
-    }
+    fun verifications(environment: DataFetchingEnvironment): CompletableFuture<List<ApplicationVerificationView>> =
+        verificationsByApplicationLoader.fromEnvironment(environment).load(id).thenApply { it!! }
 }

backend/src/main/kotlin/app/ehrenamtskarte/backend/auth/database/Setup.kt

@@ -4,7 +4,7 @@ import org.jetbrains.exposed.sql.SchemaUtils
 
 fun setupDatabase() {
     SchemaUtils.create(
-        Administrators
+        Administrators,
     )
     createEmailIndexIfNotExists()
 }

backend/src/main/kotlin/app/ehrenamtskarte/backend/auth/database/repos/AdministratorsRepository.kt

@@ -32,7 +32,7 @@ object AdministratorsRepository {
         val resultRow = (Administrators innerJoin Projects)
             .slice(Administrators.columns)
             .select(
-                (Projects.project eq project) and (LowerCase(Administrators.email) eq email.lowercase())
+                (Projects.project eq project) and (LowerCase(Administrators.email) eq email.lowercase()),
             )
             .firstOrNull()
         return resultRow?.let {
@@ -51,7 +51,7 @@ object AdministratorsRepository {
         email: String,
         password: String?,
         role: Role,
-        regionId: Int? = null
+        regionId: Int? = null,
     ): AdministratorEntity {
         val projectEntity = ProjectEntity.find { Projects.project eq project }.firstOrNull()
             ?: throw IllegalArgumentException("Project does not exist.")

backend/src/main/kotlin/app/ehrenamtskarte/backend/auth/webservice/authGraphQLParams.kt

@@ -1,33 +1,26 @@
 package app.ehrenamtskarte.backend.auth.webservice
 
-import app.ehrenamtskarte.backend.auth.webservice.dataloader.ADMINISTRATOR_LOADER_NAME
 import app.ehrenamtskarte.backend.auth.webservice.dataloader.administratorLoader
 import app.ehrenamtskarte.backend.auth.webservice.schema.ChangePasswordMutationService
 import app.ehrenamtskarte.backend.auth.webservice.schema.ManageUsersMutationService
 import app.ehrenamtskarte.backend.auth.webservice.schema.ResetPasswordMutationService
 import app.ehrenamtskarte.backend.auth.webservice.schema.SignInMutationService
 import app.ehrenamtskarte.backend.auth.webservice.schema.ViewAdministratorsQueryService
 import app.ehrenamtskarte.backend.common.webservice.GraphQLParams
+import app.ehrenamtskarte.backend.common.webservice.createRegistryFromNamedDataLoaders
 import com.expediagroup.graphql.generator.SchemaGeneratorConfig
 import com.expediagroup.graphql.generator.TopLevelObject
-import org.dataloader.DataLoaderRegistry
-
-private fun createDataLoaderRegistry(): DataLoaderRegistry {
-    val dataLoader = DataLoaderRegistry()
-    dataLoader.register(ADMINISTRATOR_LOADER_NAME, administratorLoader)
-    return dataLoader
-}
 
 val authGraphQlParams = GraphQLParams(
     config = SchemaGeneratorConfig(supportedPackages = listOf("app.ehrenamtskarte.backend.auth.webservice.schema")),
-    dataLoaderRegistry = createDataLoaderRegistry(),
+    dataLoaderRegistry = createRegistryFromNamedDataLoaders(administratorLoader),
     mutations = listOf(
         TopLevelObject(SignInMutationService()),
         TopLevelObject(ChangePasswordMutationService()),
         TopLevelObject(ResetPasswordMutationService()),
-        TopLevelObject(ManageUsersMutationService())
+        TopLevelObject(ManageUsersMutationService()),
     ),
     queries = listOf(
-        TopLevelObject(ViewAdministratorsQueryService())
-    )
+        TopLevelObject(ViewAdministratorsQueryService()),
+    ),
 )

backend/src/main/kotlin/app/ehrenamtskarte/backend/auth/webservice/dataloader/AdministratorDataLoader.kt

@@ -3,26 +3,15 @@ package app.ehrenamtskarte.backend.auth.webservice.dataloader
 import app.ehrenamtskarte.backend.auth.database.repos.AdministratorsRepository
 import app.ehrenamtskarte.backend.auth.webservice.schema.types.Administrator
 import app.ehrenamtskarte.backend.auth.webservice.schema.types.Role
-import com.expediagroup.graphql.generator.exceptions.GraphQLKotlinException
-import kotlinx.coroutines.runBlocking
-import org.dataloader.DataLoader
-import org.dataloader.DataLoaderFactory
+import app.ehrenamtskarte.backend.common.webservice.newNamedDataLoader
 import org.jetbrains.exposed.sql.transactions.transaction
-import java.util.concurrent.CompletableFuture
 
-const val ADMINISTRATOR_LOADER_NAME = "ADMINISTRATOR_LOADER"
-
-val administratorLoader: DataLoader<Int, Administrator?> = DataLoaderFactory.newDataLoader { ids ->
-    CompletableFuture.supplyAsync {
-        runBlocking {
-            transaction {
-                AdministratorsRepository.findByIds(ids).map {
-                    if (it == null) null
-                    else {
-                        val role = Role.fromDbValue(it.role) ?: throw GraphQLKotlinException("Invalid role.")
-                        Administrator(it.id.value, it.email, it.regionId?.value, role)
-                    }
-                }
+val administratorLoader = newNamedDataLoader("ADMINISTRATOR_LOADER") { ids ->
+    transaction {
+        AdministratorsRepository.findByIds(ids).map {
+            it?.let {
+                val role = Role.fromDbValue(it.role)
+                Administrator(it.id.value, it.email, it.regionId?.value, role)
             }
         }
     }

backend/src/main/kotlin/app/ehrenamtskarte/backend/common/database/sortByKeys.kt

@@ -1,6 +1,14 @@
 package app.ehrenamtskarte.backend.common.database
 
-fun <TValue, TKey> Iterable<TValue>.sortByKeys(keyFetcher: (TValue) -> TKey, keys: Iterable<TKey>): List<TValue?> {
-    val objectsMap = this.associateBy { keyFetcher(it) }
-    return keys.map { key -> objectsMap[key] }.asIterable().toList()
+/***
+ * Extending an iterable of values (this).
+ * Given a list of keys, we return `result`, a list of optional values.
+ * The i-th element in `result` corresponds to the i-th element in `keys`.
+ * If there was no value present with the i-th key, `result[i]` is null.
+ * If there was exactly one value present with the i-th key, this value will be in `result[i]`.
+ * Otherwise, if there are multiple values with the i-th key, then the method will throw.
+ */
+fun <TValue, TKey> Iterable<TValue>.sortByKeys(keySelector: (TValue) -> TKey, keys: Iterable<TKey>): List<TValue?> {
+    val valuesByKey = groupBy(keySelector)
+    return keys.map { valuesByKey[it]?.single() }
 }

backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/GraphQLParams.kt

@@ -10,15 +10,19 @@ data class GraphQLParams(
     val dataLoaderRegistry: DataLoaderRegistry,
     val queries: List<TopLevelObject>,
     val mutations: List<TopLevelObject> = emptyList(),
-    val subscriptions: List<TopLevelObject> = emptyList()
+    val subscriptions: List<TopLevelObject> = emptyList(),
 ) {
     infix fun stitch(other: GraphQLParams): GraphQLParams {
+        val duplicateDataLoaderNames = dataLoaderRegistry.keys.intersect(other.dataLoaderRegistry.keys)
+        if (duplicateDataLoaderNames.isNotEmpty()) {
+            throw IllegalArgumentException("Duplicate names for data loaders found: $duplicateDataLoaderNames")
+        }
         return GraphQLParams(
             config + other.config,
             dataLoaderRegistry.combine(other.dataLoaderRegistry),
             queries + other.queries,
             mutations + other.mutations,
-            subscriptions + other.subscriptions
+            subscriptions + other.subscriptions,
         )
     }
 }
@@ -30,6 +34,6 @@ infix operator fun SchemaGeneratorConfig.plus(other: SchemaGeneratorConfig): Sch
     val hooks = if (hooks == NoopSchemaGeneratorHooks) other.hooks else hooks
     return SchemaGeneratorConfig(
         this.supportedPackages + other.supportedPackages,
-        hooks = hooks
+        hooks = hooks,
     )
 }

backend/src/main/kotlin/app/ehrenamtskarte/backend/common/webservice/NamedDataLoader.kt

@@ -0,0 +1,44 @@
+package app.ehrenamtskarte.backend.common.webservice
+
+import graphql.schema.DataFetchingEnvironment
+import kotlinx.coroutines.runBlocking
+import org.dataloader.DataLoader
+import org.dataloader.DataLoaderFactory
+import org.dataloader.DataLoaderOptions
+import org.dataloader.DataLoaderRegistry
+import java.util.concurrent.CompletableFuture
+
+interface NamedDataLoader<K, V> {
+    val name: String
+    val loader: DataLoader<K, V>
+}
+
+fun <K, V> NamedDataLoader<K, V>.fromEnvironment(environment: DataFetchingEnvironment): DataLoader<K, V> {
+    return environment.getDataLoader(name)
+        ?: throw IllegalArgumentException("Registry does not have a DataLoader named $name")
+}
+
+fun <K, V> newNamedDataLoader(name: String, loadBatch: suspend (ids: List<K>) -> List<V>): NamedDataLoader<K, V> {
+    return object : NamedDataLoader<K, V> {
+        override val name = name
+        override val loader = DataLoaderFactory.newDataLoader(
+            { ids ->
+                CompletableFuture.supplyAsync {
+                    runBlocking { loadBatch(ids) }
+                }
+            },
+            DataLoaderOptions.newOptions().setCachingEnabled(false),
+        )
+    }
+}
+
+fun createRegistryFromNamedDataLoaders(vararg loaders: NamedDataLoader<*, *>): DataLoaderRegistry {
+    val registry = DataLoaderRegistry()
+    loaders.forEach {
+        if (it.name in registry.keys) {
+            throw IllegalArgumentException("Duplicate name of data loader specified.")
+        }
+        registry.register(it.name, it.loader)
+    }
+    return registry
+}

backend/src/main/kotlin/app/ehrenamtskarte/backend/projects/database/Setup.kt

@@ -6,7 +6,7 @@ import org.jetbrains.exposed.sql.transactions.transaction
 
 fun setupDatabase(config: BackendConfiguration) {
     SchemaUtils.create(
-        Projects
+        Projects,
     )
 
     transaction {

backend/src/main/kotlin/app/ehrenamtskarte/backend/regions/webservice/dataloader/RegionDataLoader.kt

@@ -1,24 +1,16 @@
 package app.ehrenamtskarte.backend.regions.webservice.dataloader
 
+import app.ehrenamtskarte.backend.common.webservice.newNamedDataLoader
 import app.ehrenamtskarte.backend.regions.database.repos.RegionsRepository
 import app.ehrenamtskarte.backend.regions.webservice.schema.types.Region
-import kotlinx.coroutines.runBlocking
-import org.dataloader.DataLoader
-import org.dataloader.DataLoaderFactory
 import org.jetbrains.exposed.sql.transactions.transaction
-import java.util.concurrent.CompletableFuture
 
 const val REGION_LOADER_NAME = "REGION_LOADER"
 
-val regionLoader: DataLoader<Int, Region?> = DataLoaderFactory.newDataLoader { ids ->
-    CompletableFuture.supplyAsync {
-        runBlocking {
-            transaction {
-                RegionsRepository.findByIds(ids).map {
-                    if (it == null) null
-                    else Region(it.id.value, it.prefix, it.name, it.regionIdentifier, it.dataPrivacyPolicy)
-                }
-            }
+val regionLoader = newNamedDataLoader("REGION_LOADER") { ids ->
+    transaction {
+        RegionsRepository.findByIds(ids).map {
+            it?.let { Region(it.id.value, it.prefix, it.name, it.regionIdentifier, it.dataPrivacyPolicy) }
         }
     }
 }