GH-2018 Allow to set max age for metadata files in PRIORITIZE_UPSTREA…

…M_METADATA policy (#2018) * Add "no-refetch-timeout" for StoragePolicy.PRIORITIZE_UPSTREAM_METADATA This commit adds an option to define a timeout until no re-fetch of maven-metadata.xml is done. This gives the possibility to use the local copy until the defined period is over. Especially on connections with "huge" delays this setting improves build times and still has a deprecation mechanism for maven-metadata.xml content. The default value is 0 seconds (it fetches the file every time). * GH-2018 Small refactor --------- Co-authored-by: Martin Krüger <[email protected]> Co-authored-by: dzikoysk <[email protected]>
dzikoysk · Jan 10, 2024 · e334afa · e334afa
1 parent e65eacd
commit e334afa
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 5 deletions.
diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/MirrorService.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/MirrorService.kt
@@ -18,6 +18,7 @@ package com.reposilite.maven
 
 import com.reposilite.journalist.Journalist
 import com.reposilite.journalist.Logger
+import com.reposilite.maven.StoragePolicy.PRIORITIZE_UPSTREAM_METADATA
 import com.reposilite.maven.api.METADATA_FILE
 import com.reposilite.maven.application.MirroredRepositorySettings
 import com.reposilite.shared.ErrorResponse
@@ -29,11 +30,27 @@ import panda.std.Result
 import panda.std.Result.error
 import panda.std.Result.ok
 import java.io.InputStream
+import java.time.Clock
+import java.time.Instant
+import java.time.temporal.ChronoUnit
 
-internal class MirrorService(private val journalist: Journalist) : Journalist {
+internal class MirrorService(
+    private val journalist: Journalist,
+    private val clock: Clock
+) : Journalist {
 
     fun shouldPrioritizeMirrorRepository(repository: Repository, gav: Location): Boolean =
-        repository.storagePolicy == StoragePolicy.PRIORITIZE_UPSTREAM_METADATA && gav.getSimpleName().contains(METADATA_FILE)
+        when {
+            repository.storagePolicy == PRIORITIZE_UPSTREAM_METADATA && gav.getSimpleName().contains(METADATA_FILE) ->
+                repository.metadataMaxAgeInSeconds <= 0 || !isMetadataFileValid(repository, gav)
+            else ->
+                false
+        }
+
+    private fun isMetadataFileValid(repository: Repository, gav: Location): Boolean =
+        repository.storageProvider.getLastModifiedTime(gav)
+            .map { it.toInstant().plus(repository.metadataMaxAgeInSeconds, ChronoUnit.SECONDS) }
+            .matches { it.isBefore(Instant.now(clock)) }
 
     fun findRemoteDetails(repository: Repository, gav: Location): Result<FileDetails, ErrorResponse> =
         searchInRemoteRepositories(repository, gav) { (host, config, client) ->

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/Repository.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/Repository.kt
@@ -34,7 +34,8 @@ class Repository internal constructor(
     val preserveSnapshots: Boolean,
     val mirrorHosts: List<MirrorHost>,
     val storageProvider: StorageProvider,
-    val storagePolicy: StoragePolicy
+    val storagePolicy: StoragePolicy,
+    val metadataMaxAgeInSeconds: Long
 ) {
 
     init {

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/RepositoryFactory.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/RepositoryFactory.kt
@@ -58,7 +58,8 @@ internal class RepositoryFactory(
                         storageSettings = configuration.storageProvider
                     )
                     ?: throw IllegalArgumentException("Unknown storage provider '${configuration.storageProvider.type}'"),
-            storagePolicy = configuration.storagePolicy
+            storagePolicy = configuration.storagePolicy,
+            metadataMaxAgeInSeconds = configuration.metadataMaxAge
         )
 
     private fun createMirroredHostConfiguration(configurationSource: MirroredRepositorySettings): MirrorHost? {

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/application/MavenComponents.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/application/MavenComponents.kt
@@ -34,8 +34,10 @@ import com.reposilite.storage.StorageFacade
 import com.reposilite.token.AccessTokenFacade
 import panda.std.reactive.Reference
 import java.nio.file.Path
+import java.time.Clock
 
 internal class MavenComponents(
+    private val clock: Clock,
     private val workingDirectory: Path,
     private val journalist: Journalist,
     private val extensions: Extensions,
@@ -56,7 +58,10 @@ internal class MavenComponents(
         MetadataService(securityProvider())
 
     private fun mirrorService(): MirrorService =
-        MirrorService(journalist)
+        MirrorService(
+            journalist = journalist,
+            clock = clock
+        )
 
     private fun repositoryProvider(
         mirrorService: MirrorService = mirrorService(),

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/application/MavenPlugin.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/application/MavenPlugin.kt
@@ -32,6 +32,7 @@ import com.reposilite.plugin.facade
 import com.reposilite.plugin.parameters
 import com.reposilite.shared.http.HttpRemoteClientProvider
 import com.reposilite.web.api.RoutingSetupEvent
+import java.time.Clock
 
 @Plugin(
     name = "maven",
@@ -45,6 +46,7 @@ internal class MavenPlugin : ReposilitePlugin() {
 
         val mavenFacade =
             MavenComponents(
+                clock = Clock.systemDefaultZone(),
                 workingDirectory = parameters().workingDirectory,
                 journalist = this,
                 extensions = extensions(),

diff --git a/reposilite-backend/src/main/kotlin/com/reposilite/maven/application/MavenSettings.kt b/reposilite-backend/src/main/kotlin/com/reposilite/maven/application/MavenSettings.kt
@@ -62,6 +62,11 @@ data class RepositorySettings(
         STRICT - prioritize cached version over upstream metadata (full offline mode)
     """)
     val storagePolicy: StoragePolicy = StoragePolicy.PRIORITIZE_UPSTREAM_METADATA,
+    @get:Doc(title = "Max age of metadata file", description = """
+        Fetching metadata files with PRIORITIZE_UPSTREAM_METADATA may be slow, so you can use dedicated TTL threshold for them.<br/>
+        If set to "0" then a fetch is done always. (Default: 0 seconds)
+    """)
+    val metadataMaxAge: Long = 0L,
     @get:Doc(title = "Mirrored repositories", description = "List of mirrored repositories associated with this repository.")
     val proxied: List<MirroredRepositorySettings> = listOf()
 ) : SharedSettings

diff --git a/reposilite-backend/src/test/kotlin/com/reposilite/maven/specification/MavenSpecification.kt b/reposilite-backend/src/test/kotlin/com/reposilite/maven/specification/MavenSpecification.kt
@@ -58,6 +58,7 @@ import panda.std.reactive.reference
 import panda.std.reactive.toReference
 import java.io.File
 import java.nio.file.Files
+import java.time.Clock
 
 internal abstract class MavenSpecification {
 
@@ -73,6 +74,7 @@ internal abstract class MavenSpecification {
     lateinit var workingDirectory: File
     protected lateinit var mavenFacade: MavenFacade
 
+    private val clock = Clock.systemDefaultZone()
     private val logger = InMemoryLogger()
     protected val extensions = Extensions(logger)
 
@@ -122,6 +124,7 @@ internal abstract class MavenSpecification {
         )
 
         this.mavenFacade = MavenComponents(
+            clock = clock,
             workingDirectory = workingDirectory.toPath(),
             journalist = logger,
             extensions = extensions,