From 1b13a8111a82b2e2e34e08add05c6ef7394ae2de Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Mon, 27 Nov 2023 19:57:31 +0100 Subject: [PATCH 1/6] Disable schedule for docker workflow. --- .github/workflows/docker-ort.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-ort.yml b/.github/workflows/docker-ort.yml index 1435e572864a..403bea7f4f25 100644 --- a/.github/workflows/docker-ort.yml +++ b/.github/workflows/docker-ort.yml @@ -19,8 +19,8 @@ name: ORT Docker Image on: workflow_dispatch: - schedule: - - cron: '0 4 * * *' +# schedule: +# - cron: '0 4 * * *' pull_request: paths: - '.versions' From cc740c9aec673ddb67e6c9ff8c0e8cc672789ee8 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Fri, 1 Dec 2023 23:35:45 +0100 Subject: [PATCH 2/6] feat(maven): improve speed of resolving maven dependencies. Signed-off-by: Thomas Neidhart --- .../src/main/kotlin/utils/MavenSupport.kt | 140 ++++++------------ 1 file changed, 46 insertions(+), 94 deletions(-) diff --git a/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt b/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt index 9e32aad1a027..42da7743fc98 100644 --- a/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt +++ b/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt @@ -62,6 +62,7 @@ import org.eclipse.aether.repository.MirrorSelector import org.eclipse.aether.repository.RemoteRepository import org.eclipse.aether.repository.WorkspaceReader import org.eclipse.aether.resolution.ArtifactDescriptorRequest +import org.eclipse.aether.resolution.ArtifactRequest import org.eclipse.aether.spi.connector.ArtifactDownload import org.eclipse.aether.spi.connector.layout.RepositoryLayout import org.eclipse.aether.spi.connector.layout.RepositoryLayoutProvider @@ -499,21 +500,9 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { private fun requestRemoteArtifact( artifact: Artifact, - repositories: List, - useReposFromDependencies: Boolean + artifactRepository: RemoteRepository, + allRepositories: List, ): RemoteArtifact { - val allRepositories = if (useReposFromDependencies) { - val repoSystem = containerLookup() - - // Create an artifact descriptor to get the list of repositories from the related POM file. - val artifactDescriptorRequest = ArtifactDescriptorRequest(artifact, repositories, "project") - val artifactDescriptorResult = repoSystem - .readArtifactDescriptor(repositorySystemSession, artifactDescriptorRequest) - repositories + artifactDescriptorResult.repositories - } else { - repositories - }.toSet() - val cacheKey = "$artifact@$allRepositories" remoteArtifactCache.read(cacheKey)?.let { @@ -521,22 +510,6 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { return it.fromYaml() } - // Filter out local repositories, as remote artifacts should never point to files on the local disk. - val remoteRepositories = allRepositories.filterNot { - // Some (Linux) file URIs do not start with "file://" but look like "file:/opt/android-sdk-linux". - it.url.startsWith("file:/") - }.map { repository -> - val proxy = repositorySystemSession.proxySelector.getProxy(repository) - val authentication = repositorySystemSession.authenticationSelector.getAuthentication(repository) - RemoteRepository.Builder(repository).setAuthentication(authentication).setProxy(proxy).build() - }.toSet() - - if (allRepositories.size > remoteRepositories.size) { - logger.debug { "Ignoring local repositories ${allRepositories - remoteRepositories}." } - } - - logger.debug { "Searching for '$artifact' in $remoteRepositories." } - data class ArtifactLocationInfo( val repository: RemoteRepository, val layout: RepositoryLayout, @@ -546,7 +519,7 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { val repositoryLayoutProvider = containerLookup() - val locationInfo = remoteRepositories.mapNotNull { repository -> + val locationInfo = listOf(artifactRepository).mapNotNull { repository -> val repositoryLayout = runCatching { repositoryLayoutProvider.newRepositoryLayout(repositorySystemSession, repository) }.onFailure { @@ -562,8 +535,6 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { } } - val remoteRepositoryManager = containerLookup() - val repositoryConnectorProvider = containerLookup() val transporterProvider = containerLookup() // Check the remote repositories for the availability of the artifact. @@ -571,73 +542,32 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { locationInfo.forEach { info -> logger.debug { "Trying to download artifact '$artifact' from ${info.downloadUrl}." } - val snapshot = artifact.isSnapshot - val policy = remoteRepositoryManager.getPolicy( - repositorySystemSession, info.repository, !snapshot, snapshot - ) - - val localPath = repositorySystemSession.localRepositoryManager - .getPathForRemoteArtifact(artifact, info.repository, "project") - val downloadFile = File(repositorySystemSession.localRepositoryManager.repository.basedir, localPath) - - val artifactDownload = ArtifactDownload(artifact, "project", downloadFile, policy.checksumPolicy) - artifactDownload.isExistenceCheck = true - artifactDownload.listener = object : AbstractTransferListener() { - override fun transferFailed(event: TransferEvent?) { - logger.debug { - "Transfer failed for repository with ID '${info.repository.id}': $event" - } - } + val checksumLocations = info.layout.getChecksumLocations(artifact, false, info.location) - override fun transferSucceeded(event: TransferEvent?) { - logger.debug { "Transfer succeeded: $event" } - } - } + // TODO: Could store multiple checksums in model instead of only the first. + val checksumLocation = checksumLocations.first() - try { - wrapMavenSession { - repositoryConnectorProvider.newRepositoryConnector(repositorySystemSession, info.repository).use { - it.get(listOf(artifactDownload), null) - } - } - } catch (e: NoRepositoryConnectorException) { - e.showStackTrace() + logger.info { "Getting checksum for '$artifact' in '${info.repository}' ${checksumLocation}." } - logger.warn { "Could not create connector for repository '${info.repository}': ${e.collectMessages()}" } + val transporter = transporterProvider.newTransporter(repositorySystemSession, info.repository) - return@forEach - } - - if (artifactDownload.exception == null) { - logger.debug { "Found '$artifact' in '${info.repository}'." } - - val checksumLocations = info.layout.getChecksumLocations(artifact, false, info.location) - - // TODO: Could store multiple checksums in model instead of only the first. - val checksumLocation = checksumLocations.first() + val hash = runCatching { + val task = GetTask(checksumLocation.location) + transporter.get(task) - val transporter = transporterProvider.newTransporter(repositorySystemSession, info.repository) - - val hash = runCatching { - val task = GetTask(checksumLocation.location) - transporter.get(task) - - parseChecksum(task.dataString, checksumLocation.checksumAlgorithmFactory.name) - }.getOrElse { - it.showStackTrace() + parseChecksum(task.dataString, checksumLocation.checksumAlgorithmFactory.name) + }.getOrElse { + it.showStackTrace() - logger.warn { "Could not get checksum for '$artifact': ${it.collectMessages()}" } + logger.warn { "Could not get checksum for '$artifact': ${it.collectMessages()}" } - // Fall back to an empty hash. - Hash.NONE - } + // Fall back to an empty hash. + Hash.NONE + } - return RemoteArtifact(info.downloadUrl, hash).also { remoteArtifact -> - logger.debug { "Writing remote artifact for '$artifact' to disk cache." } - remoteArtifactCache.write(cacheKey, remoteArtifact.toYaml()) - } - } else { - logger.debug { artifactDownload.exception.collectMessages() } + return RemoteArtifact(info.downloadUrl, hash).also { remoteArtifact -> + logger.debug { "Writing remote artifact for '$artifact' to disk cache." } + remoteArtifactCache.write(cacheKey, remoteArtifact.toYaml()) } } @@ -719,9 +649,31 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { val declaredLicenses = parseLicenses(mavenProject) val declaredLicensesProcessed = processDeclaredLicenses(declaredLicenses) + // retrieve the pom file for the artifact to know from which repository it is coming + val pomArtifact = artifact.let { + DefaultArtifact(it.groupId, it.artifactId, it.classifier, "pom", it.version) + } + val pomArtifactRequest = ArtifactRequest(pomArtifact, repositories, "project") + val result = containerLookup().resolveArtifact(repositorySystemSession, pomArtifactRequest) + val artifactRepository = result.repository as RemoteRepository + + // reconstruct all repositories that would be used for resolving the artifact + // the repositories are used to construct a cache key for the artifact + val allRepositories = if (useReposFromDependencies) { + val repoSystem = containerLookup() + + // Create an artifact descriptor to get the list of repositories from the related POM file. + val artifactDescriptorRequest = ArtifactDescriptorRequest(artifact, repositories, "project") + val artifactDescriptorResult = repoSystem + .readArtifactDescriptor(repositorySystemSession, artifactDescriptorRequest) + repositories + artifactDescriptorResult.repositories + } else { + repositories + }.toSet() + val binaryRemoteArtifact = localProject?.let { RemoteArtifact.EMPTY - } ?: requestRemoteArtifact(artifact, repositories, useReposFromDependencies) + } ?: requestRemoteArtifact(artifact, artifactRepository, allRepositories.toList()) val isBinaryArtifactModified = isArtifactModified(artifact, binaryRemoteArtifact) @@ -733,7 +685,7 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { DefaultArtifact(it.groupId, it.artifactId, "sources", "jar", it.version) } - requestRemoteArtifact(sourceArtifact, repositories, useReposFromDependencies) + requestRemoteArtifact(sourceArtifact, artifactRepository, allRepositories.toList()) } } From c12b601d92cb2529cd26836b6f590dcc44d65399 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Fri, 1 Dec 2023 23:44:46 +0100 Subject: [PATCH 3/6] Revert temp change. --- .github/workflows/docker-ort.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-ort.yml b/.github/workflows/docker-ort.yml index 403bea7f4f25..1435e572864a 100644 --- a/.github/workflows/docker-ort.yml +++ b/.github/workflows/docker-ort.yml @@ -19,8 +19,8 @@ name: ORT Docker Image on: workflow_dispatch: -# schedule: -# - cron: '0 4 * * *' + schedule: + - cron: '0 4 * * *' pull_request: paths: - '.versions' From 86b2db14916f539dc08614ec5ae3868eb7e07dd7 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Fri, 1 Dec 2023 23:44:58 +0100 Subject: [PATCH 4/6] Cleanup unused imports. --- .../maven/src/main/kotlin/utils/MavenSupport.kt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt b/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt index 42da7743fc98..f0d2d07d66a1 100644 --- a/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt +++ b/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt @@ -56,21 +56,15 @@ import org.eclipse.aether.RepositorySystem import org.eclipse.aether.RepositorySystemSession import org.eclipse.aether.artifact.Artifact import org.eclipse.aether.artifact.DefaultArtifact -import org.eclipse.aether.impl.RemoteRepositoryManager -import org.eclipse.aether.impl.RepositoryConnectorProvider import org.eclipse.aether.repository.MirrorSelector import org.eclipse.aether.repository.RemoteRepository import org.eclipse.aether.repository.WorkspaceReader import org.eclipse.aether.resolution.ArtifactDescriptorRequest import org.eclipse.aether.resolution.ArtifactRequest -import org.eclipse.aether.spi.connector.ArtifactDownload import org.eclipse.aether.spi.connector.layout.RepositoryLayout import org.eclipse.aether.spi.connector.layout.RepositoryLayoutProvider import org.eclipse.aether.spi.connector.transport.GetTask import org.eclipse.aether.spi.connector.transport.TransporterProvider -import org.eclipse.aether.transfer.AbstractTransferListener -import org.eclipse.aether.transfer.NoRepositoryConnectorException -import org.eclipse.aether.transfer.TransferEvent import org.eclipse.aether.util.repository.JreProxySelector import org.ossreviewtoolkit.analyzer.PackageManager From 072b1ae45370a411f695801169affbf20058b2c1 Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Fri, 1 Dec 2023 23:53:13 +0100 Subject: [PATCH 5/6] Check if artifact exists if no checksum is found. --- .../maven/src/main/kotlin/utils/MavenSupport.kt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt b/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt index f0d2d07d66a1..f55c7237ef5d 100644 --- a/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt +++ b/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt @@ -64,6 +64,7 @@ import org.eclipse.aether.resolution.ArtifactRequest import org.eclipse.aether.spi.connector.layout.RepositoryLayout import org.eclipse.aether.spi.connector.layout.RepositoryLayoutProvider import org.eclipse.aether.spi.connector.transport.GetTask +import org.eclipse.aether.spi.connector.transport.PeekTask import org.eclipse.aether.spi.connector.transport.TransporterProvider import org.eclipse.aether.util.repository.JreProxySelector @@ -559,6 +560,21 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { Hash.NONE } + if (hash == Hash.NONE) { + val artifactFound = runCatching { + val task = PeekTask(info.location) + transporter.peek(task) + true + }.getOrElse { + it.showStackTrace() + logger.warn { "Could not get artifact '$artifact': ${it.collectMessages()}" } + false + } + + if (!artifactFound) { + return@forEach + } + } return RemoteArtifact(info.downloadUrl, hash).also { remoteArtifact -> logger.debug { "Writing remote artifact for '$artifact' to disk cache." } remoteArtifactCache.write(cacheKey, remoteArtifact.toYaml()) From 39f1284ea0d4b6db8ebc00535d6f219fb5e540bd Mon Sep 17 00:00:00 2001 From: Thomas Neidhart Date: Sat, 2 Dec 2023 00:23:43 +0100 Subject: [PATCH 6/6] When retrieving pom for the artifact do not include a classifier, ignore artifacts that are resolved locally. --- .../maven/src/main/kotlin/utils/MavenSupport.kt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt b/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt index f55c7237ef5d..d86ca5071759 100644 --- a/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt +++ b/plugins/package-managers/maven/src/main/kotlin/utils/MavenSupport.kt @@ -661,11 +661,11 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { // retrieve the pom file for the artifact to know from which repository it is coming val pomArtifact = artifact.let { - DefaultArtifact(it.groupId, it.artifactId, it.classifier, "pom", it.version) + DefaultArtifact(it.groupId, it.artifactId, null, "pom", it.version) } val pomArtifactRequest = ArtifactRequest(pomArtifact, repositories, "project") val result = containerLookup().resolveArtifact(repositorySystemSession, pomArtifactRequest) - val artifactRepository = result.repository as RemoteRepository + val artifactRepository = result.repository // reconstruct all repositories that would be used for resolving the artifact // the repositories are used to construct a cache key for the artifact @@ -681,14 +681,19 @@ class MavenSupport(private val workspaceReader: WorkspaceReader) { repositories }.toSet() - val binaryRemoteArtifact = localProject?.let { - RemoteArtifact.EMPTY - } ?: requestRemoteArtifact(artifact, artifactRepository, allRepositories.toList()) + val binaryRemoteArtifact = when { + localProject != null -> RemoteArtifact.EMPTY + artifactRepository !is RemoteRepository -> RemoteArtifact.EMPTY + else -> { + requestRemoteArtifact(artifact, artifactRepository, allRepositories.toList()) + } + } val isBinaryArtifactModified = isArtifactModified(artifact, binaryRemoteArtifact) val sourceRemoteArtifact = when { localProject != null -> RemoteArtifact.EMPTY + artifactRepository !is RemoteRepository -> RemoteArtifact.EMPTY artifact.extension == "pom" -> binaryRemoteArtifact else -> { val sourceArtifact = artifact.let {