From bf70534d6942b6812ccf9e2c67aea9155530b2f1 Mon Sep 17 00:00:00 2001 From: nscuro Date: Fri, 9 Aug 2024 19:04:50 +0200 Subject: [PATCH] Improve efficiency of repository and integrity meta analysis * Removes "preparation" logic of `IntegrityMetaComponent` records in `BomUploadProcessingTask`. Preparing records one-by-one is too resource-intensive, but doing it in batches has a too high potential for deadlocks since the table is written to by many threads in parallel. * Refactors `RepositoryMetaResultProcessor` to consume records in batches. Since incoming records are keyed by PURL coordinates, we can safely perform batch operations in the database without the risk of running into deadlocks. Signed-off-by: nscuro --- .../org/dependencytrack/common/ConfigKey.java | 3 - .../ComponentRepositoryMetaAnalysisEvent.java | 12 +- .../event/EventSubsystemInitializer.java | 8 +- .../event/IntegrityAnalysisEvent.java | 46 --- .../event/IntegrityMetaInitializerEvent.java | 24 -- .../dependencytrack/event/PurlMigrator.java | 66 ---- .../event/kafka/KafkaEventConverter.java | 23 +- .../componentmeta/AbstractMetaHandler.java | 46 --- .../componentmeta/ComponentProjection.java | 26 -- .../event/kafka/componentmeta/Handler.java | 26 -- .../kafka/componentmeta/HandlerFactory.java | 36 -- .../kafka/componentmeta/IntegrityCheck.java | 134 -------- .../componentmeta/RepoMetaConstants.java | 27 -- .../componentmeta/SupportedMetaHandler.java | 68 ---- .../componentmeta/UnSupportedMetaHandler.java | 43 --- .../kafka/processor/ProcessorInitializer.java | 2 +- .../RepositoryMetaResultProcessor.java | 303 ++++++++++------- .../kafka/processor/api/ProcessorUtils.java | 88 +++++ .../model/RepositoryMetaComponent.java | 15 + .../persistence/ComponentQueryManager.java | 23 +- .../IntegrityAnalysisQueryManager.java | 44 --- .../IntegrityMetaQueryManager.java | 211 ------------ .../persistence/QueryManager.java | 59 ---- .../persistence/jdbi/MetaComponentDao.java | 230 +++++++++++++ .../resources/v1/ComponentResource.java | 307 +++++++----------- .../tasks/BomUploadProcessingTask.java | 101 ++---- .../tasks/IntegrityAnalysisTask.java | 57 ---- .../tasks/IntegrityMetaInitializerTask.java | 92 ------ .../org/dependencytrack/tasks/LockName.java | 1 - .../tasks/RepositoryMetaAnalyzerTask.java | 20 +- .../dependencytrack/tasks/TaskScheduler.java | 5 +- .../dependencytrack/util/LockProvider.java | 7 - .../v1/repo_meta_analysis.proto | 28 +- src/main/resources/application.properties | 19 +- .../resources/migration/changelog-main.xml | 1 + .../migration/changelog-procedures.xml | 6 + .../resources/migration/changelog-v5.6.0.xml | 32 ++ ...ction_component-integrity-check-status.sql | 19 ++ .../function_match-component-hash.sql | 17 + src/main/webapp/WEB-INF/web.xml | 3 - .../event/PurlMigratorTest.java | 89 ----- .../event/kafka/KafkaEventDispatcherTest.java | 9 +- .../componentmeta/HandlerFactoryTest.java | 67 ---- .../SupportedMetaHandlerTest.java | 125 ------- .../UnSupportedMetaHandlerTest.java | 59 ---- .../RepositoryMetaResultProcessorTest.java | 110 ++++--- ...IntegrityMetaQueryManagerPostgresTest.java | 82 ----- .../IntegrityMetaQueryManagerTest.java | 87 ----- .../persistence/QueryManagerTest.java | 6 +- .../policy/cel/CelPolicyEngineTest.java | 4 +- .../cel/compat/ComponentAgeCelPolicyTest.java | 2 +- .../resources/v1/ComponentResourceTest.java | 130 +------- .../tasks/BomUploadProcessingTaskTest.java | 2 +- .../tasks/IntegrityAnalysisTaskTest.java | 114 ------- .../IntegrityMetaInitializerTaskTest.java | 125 ------- .../tasks/RepositoryMetaAnalyzerTaskTest.java | 46 ++- 56 files changed, 931 insertions(+), 2404 deletions(-) delete mode 100644 src/main/java/org/dependencytrack/event/IntegrityAnalysisEvent.java delete mode 100644 src/main/java/org/dependencytrack/event/IntegrityMetaInitializerEvent.java delete mode 100644 src/main/java/org/dependencytrack/event/PurlMigrator.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/IntegrityCheck.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java delete mode 100644 src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java create mode 100644 src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorUtils.java delete mode 100644 src/main/java/org/dependencytrack/persistence/IntegrityAnalysisQueryManager.java delete mode 100644 src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java create mode 100644 src/main/java/org/dependencytrack/persistence/jdbi/MetaComponentDao.java delete mode 100644 src/main/java/org/dependencytrack/tasks/IntegrityAnalysisTask.java delete mode 100644 src/main/java/org/dependencytrack/tasks/IntegrityMetaInitializerTask.java create mode 100644 src/main/resources/migration/changelog-v5.6.0.xml create mode 100644 src/main/resources/migration/procedures/function_component-integrity-check-status.sql create mode 100644 src/main/resources/migration/procedures/function_match-component-hash.sql delete mode 100644 src/test/java/org/dependencytrack/event/PurlMigratorTest.java delete mode 100644 src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java delete mode 100644 src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java delete mode 100644 src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java delete mode 100644 src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerPostgresTest.java delete mode 100644 src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java delete mode 100644 src/test/java/org/dependencytrack/tasks/IntegrityAnalysisTaskTest.java delete mode 100644 src/test/java/org/dependencytrack/tasks/IntegrityMetaInitializerTaskTest.java diff --git a/src/main/java/org/dependencytrack/common/ConfigKey.java b/src/main/java/org/dependencytrack/common/ConfigKey.java index 0b7116812..6b71f3a59 100644 --- a/src/main/java/org/dependencytrack/common/ConfigKey.java +++ b/src/main/java/org/dependencytrack/common/ConfigKey.java @@ -79,9 +79,6 @@ public enum ConfigKey implements Config.Key { WORKFLOW_RETENTION_DURATION("workflow.retention.duration", "P3D"), WORKFLOW_STEP_TIMEOUT_DURATION("workflow.step.timeout.duration", "PT1H"), TMP_DELAY_BOM_PROCESSED_NOTIFICATION("tmp.delay.bom.processed.notification", "false"), - INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR("integrityMetaInitializer.lockAtMostForInMillis", String.valueOf(Duration.ofMinutes(15).toMillis())), - INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR("integrityMetaInitializer.lockAtLeastForInMillis", String.valueOf(Duration.ofMinutes(5).toMillis())), - INTEGRITY_INITIALIZER_ENABLED("integrity.initializer.enabled", "false"), INTEGRITY_CHECK_ENABLED("integrity.check.enabled", "false"), VULNERABILITY_POLICY_ANALYSIS_ENABLED("vulnerability.policy.analysis.enabled", false), VULNERABILITY_POLICY_BUNDLE_URL("vulnerability.policy.bundle.url", null), diff --git a/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java b/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java index 62419b9a4..3ac727898 100644 --- a/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java +++ b/src/main/java/org/dependencytrack/event/ComponentRepositoryMetaAnalysisEvent.java @@ -19,19 +19,15 @@ package org.dependencytrack.event; import alpine.event.framework.Event; +import com.github.packageurl.PackageURL; import org.dependencytrack.model.Component; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; - -import java.util.UUID; /** * Defines an {@link Event} triggered when requesting a component to be analyzed for meta information. * - * @param purlCoordinates The package URL coordinates of the {@link Component} to analyze - * @param internal Whether the {@link Component} is internal - * @param fetchMeta Whether component hash data or component meta data needs to be fetched from external api + * @param purl The {@link PackageURL} of the {@link Component} to analyze + * @param internal Whether the {@link Component} is internal */ -public record ComponentRepositoryMetaAnalysisEvent(UUID componentUuid, String purlCoordinates, Boolean internal, - FetchMeta fetchMeta) implements Event { +public record ComponentRepositoryMetaAnalysisEvent(PackageURL purl, Boolean internal) implements Event { } diff --git a/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java b/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java index 61d48bf5a..2f2ff4f93 100644 --- a/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java +++ b/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java @@ -30,8 +30,6 @@ import org.dependencytrack.tasks.EpssMirrorTask; import org.dependencytrack.tasks.FortifySscUploadTask; import org.dependencytrack.tasks.GitHubAdvisoryMirrorTask; -import org.dependencytrack.tasks.IntegrityAnalysisTask; -import org.dependencytrack.tasks.IntegrityMetaInitializerTask; import org.dependencytrack.tasks.InternalComponentIdentificationTask; import org.dependencytrack.tasks.KennaSecurityUploadTask; import org.dependencytrack.tasks.LdapSyncTaskWrapper; @@ -42,12 +40,12 @@ import org.dependencytrack.tasks.TaskScheduler; import org.dependencytrack.tasks.VexUploadProcessingTask; import org.dependencytrack.tasks.VulnerabilityAnalysisTask; -import org.dependencytrack.tasks.vulnerabilitypolicy.VulnerabilityPolicyFetchTask; import org.dependencytrack.tasks.VulnerabilityScanCleanupTask; import org.dependencytrack.tasks.WorkflowStateCleanupTask; import org.dependencytrack.tasks.metrics.PortfolioMetricsUpdateTask; import org.dependencytrack.tasks.metrics.ProjectMetricsUpdateTask; import org.dependencytrack.tasks.metrics.VulnerabilityMetricsUpdateTask; +import org.dependencytrack.tasks.vulnerabilitypolicy.VulnerabilityPolicyFetchTask; import jakarta.servlet.ServletContextEvent; import jakarta.servlet.ServletContextListener; @@ -100,8 +98,6 @@ public void contextInitialized(final ServletContextEvent event) { EVENT_SERVICE.subscribe(ComponentPolicyEvaluationEvent.class, PolicyEvaluationTask.class); EVENT_SERVICE.subscribe(ProjectPolicyEvaluationEvent.class, PolicyEvaluationTask.class); EVENT_SERVICE.subscribe(WorkflowStateCleanupEvent.class, WorkflowStateCleanupTask.class); - EVENT_SERVICE.subscribe(IntegrityMetaInitializerEvent.class, IntegrityMetaInitializerTask.class); - EVENT_SERVICE.subscribe(IntegrityAnalysisEvent.class, IntegrityAnalysisTask.class); TaskScheduler.getInstance(); } @@ -135,8 +131,6 @@ public void contextDestroyed(final ServletContextEvent event) { EVENT_SERVICE.unsubscribe(EpssMirrorTask.class); EVENT_SERVICE.unsubscribe(PolicyEvaluationTask.class); EVENT_SERVICE.unsubscribe(WorkflowStateCleanupTask.class); - EVENT_SERVICE.unsubscribe(IntegrityMetaInitializerTask.class); - EVENT_SERVICE.unsubscribe(IntegrityAnalysisTask.class); EVENT_SERVICE.unsubscribe(VulnerabilityPolicyFetchTask.class); EVENT_SERVICE.shutdown(DRAIN_TIMEOUT_DURATION); } diff --git a/src/main/java/org/dependencytrack/event/IntegrityAnalysisEvent.java b/src/main/java/org/dependencytrack/event/IntegrityAnalysisEvent.java deleted file mode 100644 index 97952f1a8..000000000 --- a/src/main/java/org/dependencytrack/event/IntegrityAnalysisEvent.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event; - -import alpine.event.framework.Event; -import org.dependencytrack.model.IntegrityMetaComponent; - -import java.util.UUID; - -import static org.dependencytrack.util.PersistenceUtil.assertNonPersistent; - -public class IntegrityAnalysisEvent implements Event { - - private final UUID uuid; - private final IntegrityMetaComponent integrityMetaComponent; - - public IntegrityAnalysisEvent(UUID uuid, IntegrityMetaComponent integrityMetaComponent) { - assertNonPersistent(integrityMetaComponent, "integrityMetaComponent must not be persistent"); - this.uuid = uuid; - this.integrityMetaComponent = integrityMetaComponent; - } - - public UUID getUuid() { - return uuid; - } - - public IntegrityMetaComponent getIntegrityMetaComponent() { - return integrityMetaComponent; - } -} diff --git a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializerEvent.java b/src/main/java/org/dependencytrack/event/IntegrityMetaInitializerEvent.java deleted file mode 100644 index b500b897b..000000000 --- a/src/main/java/org/dependencytrack/event/IntegrityMetaInitializerEvent.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event; - -import alpine.event.framework.Event; - -public class IntegrityMetaInitializerEvent implements Event { -} diff --git a/src/main/java/org/dependencytrack/event/PurlMigrator.java b/src/main/java/org/dependencytrack/event/PurlMigrator.java deleted file mode 100644 index de609cd11..000000000 --- a/src/main/java/org/dependencytrack/event/PurlMigrator.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event; - -import alpine.Config; -import alpine.common.logging.Logger; -import net.javacrumbs.shedlock.core.LockingTaskExecutor; -import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.util.LockProvider; - -import jakarta.servlet.ServletContextEvent; -import jakarta.servlet.ServletContextListener; - -import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_LOCK; - -public class PurlMigrator implements ServletContextListener { - - private static final Logger LOGGER = Logger.getLogger(PurlMigrator.class); - private final boolean integrityInitializerEnabled; - - public PurlMigrator() { - this(Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_INITIALIZER_ENABLED)); - } - - PurlMigrator(final boolean integrityInitializerEnabled) { - this.integrityInitializerEnabled = integrityInitializerEnabled; - } - - - @Override - public void contextInitialized(final ServletContextEvent event) { - if (integrityInitializerEnabled) { - try { - LockProvider.executeWithLock(INTEGRITY_META_INITIALIZER_LOCK, (LockingTaskExecutor.Task) () -> process()); - } catch (Throwable e) { - throw new RuntimeException("An unexpected error occurred while running Initializer for integrity meta", e); - } - } else { - LOGGER.info("Component integrity initializer is disabled."); - } - } - - private void process() { - LOGGER.info("Initializing integrity meta component sync"); - try (final var qm = new QueryManager()) { - qm.synchronizeIntegrityMetaComponent(); - } - } -} diff --git a/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java b/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java index e2cd33e17..1420760ba 100644 --- a/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java +++ b/src/main/java/org/dependencytrack/event/kafka/KafkaEventConverter.java @@ -19,6 +19,7 @@ package org.dependencytrack.event.kafka; import alpine.event.framework.Event; +import com.github.packageurl.MalformedPackageURLException; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; @@ -52,8 +53,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.UUID; +import static com.github.packageurl.PackageURLBuilder.aPackageURL; import static org.apache.commons.lang3.ObjectUtils.requireNonEmpty; /** @@ -130,21 +131,31 @@ static KafkaEvent convert(final ComponentVulnerabilityAnal } static KafkaEvent convert(final ComponentRepositoryMetaAnalysisEvent event) { - if (event == null || event.purlCoordinates() == null) { + if (event == null || event.purl() == null) { return null; } + + final String key; + try { + key = aPackageURL() + .withType(event.purl().getType()) + .withNamespace(event.purl().getNamespace()) + .withName(event.purl().getName()) + .build() + .toString(); + } catch (MalformedPackageURLException e) { + throw new IllegalStateException("Failed to build PURL coordinates without version", e); + } final var componentBuilder = org.dependencytrack.proto.repometaanalysis.v1.Component.newBuilder() - .setPurl(event.purlCoordinates()); + .setPurl(event.purl().toString()); Optional.ofNullable(event.internal()).ifPresent(componentBuilder::setInternal); - Optional.ofNullable(event.componentUuid()).map(UUID::toString).ifPresent(componentBuilder::setUuid); final var analysisCommand = AnalysisCommand.newBuilder() .setComponent(componentBuilder) - .setFetchMeta(event.fetchMeta()) .build(); - return new KafkaEvent<>(KafkaTopics.REPO_META_ANALYSIS_COMMAND, event.purlCoordinates(), analysisCommand, null); + return new KafkaEvent<>(KafkaTopics.REPO_META_ANALYSIS_COMMAND, key, analysisCommand, null); } static KafkaEvent convert(final GitHubAdvisoryMirrorEvent ignored) { diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java deleted file mode 100644 index d8aadc181..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/AbstractMetaHandler.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.model.FetchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; - -import java.time.Instant; -import java.util.Date; - -public abstract class AbstractMetaHandler implements Handler { - - ComponentProjection componentProjection; - QueryManager queryManager; - KafkaEventDispatcher kafkaEventDispatcher; - FetchMeta fetchMeta; - - - public static IntegrityMetaComponent createIntegrityMetaComponent(String purl) { - IntegrityMetaComponent integrityMetaComponent = new IntegrityMetaComponent(); - integrityMetaComponent.setStatus(FetchStatus.IN_PROGRESS); - integrityMetaComponent.setPurl(purl); - integrityMetaComponent.setLastFetch(Date.from(Instant.now())); - return integrityMetaComponent; - } - -} \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java deleted file mode 100644 index d7ca8ad6c..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/ComponentProjection.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import com.github.packageurl.PackageURL; - -import java.util.UUID; - -public record ComponentProjection(UUID componentUuid, String purlCoordinates, Boolean internal, PackageURL purl) { -} \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java deleted file mode 100644 index 33d64f309..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/Handler.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import com.github.packageurl.MalformedPackageURLException; -import org.dependencytrack.model.IntegrityMetaComponent; - -public interface Handler { - IntegrityMetaComponent handle() throws MalformedPackageURLException; -} \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java deleted file mode 100644 index 6cf7831e8..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactory.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import com.github.packageurl.MalformedPackageURLException; -import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; - -public class HandlerFactory { - - public static Handler createHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, FetchMeta fetchMeta) throws MalformedPackageURLException { - boolean result = RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(componentProjection.purl().getType()); - if (result) { - return new SupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, fetchMeta); - } else { - return new UnSupportedMetaHandler(componentProjection, queryManager, kafkaEventDispatcher, FetchMeta.FETCH_META_LATEST_VERSION); - } - } -} \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/IntegrityCheck.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/IntegrityCheck.java deleted file mode 100644 index 6fd46d526..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/IntegrityCheck.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import alpine.Config; -import alpine.common.logging.Logger; -import org.apache.commons.lang3.StringUtils; -import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.IntegrityAnalysis; -import org.dependencytrack.model.IntegrityMatchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.proto.repometaanalysis.v1.AnalysisResult; - -import java.util.Date; -import java.util.List; - -import static org.dependencytrack.model.IntegrityMatchStatus.COMPONENT_MISSING_HASH; -import static org.dependencytrack.model.IntegrityMatchStatus.COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN; -import static org.dependencytrack.model.IntegrityMatchStatus.HASH_MATCH_FAILED; -import static org.dependencytrack.model.IntegrityMatchStatus.HASH_MATCH_PASSED; -import static org.dependencytrack.model.IntegrityMatchStatus.HASH_MATCH_UNKNOWN; - -public class IntegrityCheck { - - private static final Logger LOGGER = Logger.getLogger(IntegrityCheck.class); - - public static void performIntegrityCheck(final IntegrityMetaComponent integrityMetaComponent, final AnalysisResult result, final QueryManager qm) { - if (!Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_CHECK_ENABLED)) { - LOGGER.debug("Integrity check is disabled"); - return; - } - //if integritymeta is in result with hashses but component uuid is not present, result has integrity data for existing - // components. Get components from database and perform integrity check - if (StringUtils.isBlank(result.getComponent().getUuid())) { - integrityAnalysisOfExistingComponents(integrityMetaComponent, qm); - return; - } - //check if the object is not null - final Component component = qm.getObjectByUuid(Component.class, result.getComponent().getUuid()); - if (component == null) { - LOGGER.debug("Component is not present in database for which Integrity metadata is received so skipping analysis"); - return; - } - calculateIntegrityResult(integrityMetaComponent, component, qm); - } - - private static void integrityAnalysisOfExistingComponents(final IntegrityMetaComponent integrityMetaComponent, final QueryManager qm) { - if (integrityMetaComponent != null) { - List componentList = qm.getComponentsByPurl(integrityMetaComponent.getPurl()); - for (Component component : componentList) { - LOGGER.debug("calculate integrity for component : " + component.getUuid()); - calculateIntegrityResult(integrityMetaComponent, component, qm); - } - } - } - - private static IntegrityMatchStatus checkHash(String metadataHash, String componentHash) { - if (StringUtils.isBlank(metadataHash) && StringUtils.isBlank(componentHash)) { - return COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN; - } - if (StringUtils.isBlank(metadataHash)) { - return HASH_MATCH_UNKNOWN; - } - if (StringUtils.isBlank(componentHash)) { - return COMPONENT_MISSING_HASH; - } - return componentHash.equals(metadataHash) ? HASH_MATCH_PASSED : HASH_MATCH_FAILED; - } - - public static void calculateIntegrityResult(final IntegrityMetaComponent integrityMetaComponent, final Component component, final QueryManager qm) { - //if integritymetacomponent is null, try to get it from db - //it could be that integrity metadata is already in db - IntegrityMetaComponent metadata = integrityMetaComponent == null ? qm.getIntegrityMetaComponent(component.getPurl().toString()) : integrityMetaComponent; - if (metadata == null) { - LOGGER.info("Integrity metadata is null in result and db. Cannot perform integrity analysis"); - return; - } - IntegrityMatchStatus md5Status = checkHash(metadata.getMd5(), component.getMd5()); - IntegrityMatchStatus sha1Status = checkHash(metadata.getSha1(), component.getSha1()); - IntegrityMatchStatus sha256Status = checkHash(metadata.getSha256(), component.getSha256()); - IntegrityMatchStatus sha512Status = checkHash(metadata.getSha512(), component.getSha512()); - - IntegrityAnalysis integrityAnalysis = qm.getIntegrityAnalysisByComponentUuid(component.getUuid()); - if (integrityAnalysis == null) { - integrityAnalysis = new IntegrityAnalysis(); - integrityAnalysis.setComponent(component); - } - integrityAnalysis.setIntegrityCheckStatus(calculateIntegrityCheckStatus(md5Status, sha1Status, sha256Status, sha512Status)); - integrityAnalysis.setMd5HashMatchStatus(md5Status); - integrityAnalysis.setSha1HashMatchStatus(sha1Status); - integrityAnalysis.setSha256HashMatchStatus(sha256Status); - integrityAnalysis.setSha512HashMatchStatus(sha512Status); - integrityAnalysis.setUpdatedAt(new Date()); - qm.persist(integrityAnalysis); - } - - private static IntegrityMatchStatus calculateIntegrityCheckStatus(IntegrityMatchStatus md5Status, IntegrityMatchStatus sha1Status, IntegrityMatchStatus sha256Status, IntegrityMatchStatus sha512Status) { - if (md5Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN && sha1Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN && sha256Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN && sha512Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN) { - return COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN; - } else if ((md5Status == COMPONENT_MISSING_HASH || md5Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN) - && (sha1Status == COMPONENT_MISSING_HASH || sha1Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN) - && (sha256Status == COMPONENT_MISSING_HASH || sha256Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN) - && (sha512Status == COMPONENT_MISSING_HASH || sha512Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN)) { - return COMPONENT_MISSING_HASH; - } else if ((md5Status == HASH_MATCH_UNKNOWN || md5Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN) - && (sha1Status == HASH_MATCH_UNKNOWN || sha1Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN) - && (sha256Status == HASH_MATCH_UNKNOWN || sha256Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN) - && (sha512Status == HASH_MATCH_UNKNOWN || sha512Status == COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN)) { - return HASH_MATCH_UNKNOWN; - } else if (md5Status == HASH_MATCH_FAILED || sha1Status == HASH_MATCH_FAILED || sha256Status == HASH_MATCH_FAILED || sha512Status == HASH_MATCH_FAILED) { - return HASH_MATCH_FAILED; - } else { - return HASH_MATCH_PASSED; - } - } -} diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java deleted file mode 100644 index 22c4ed960..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/RepoMetaConstants.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import java.util.List; - -public class RepoMetaConstants { - - public static final long TIME_SPAN = 60 * 60 * 1000L; - public static final List SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK =List.of("maven", "npm", "pypi"); -} \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java deleted file mode 100644 index f90004813..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandler.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import com.github.packageurl.MalformedPackageURLException; -import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; -import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.model.FetchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; - -import java.time.Instant; -import java.util.Date; - -import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.TIME_SPAN; -import static org.dependencytrack.model.FetchStatus.NOT_AVAILABLE; -import static org.dependencytrack.model.FetchStatus.PROCESSED; - -public class SupportedMetaHandler extends AbstractMetaHandler { - - public SupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, FetchMeta fetchMeta) { - this.componentProjection = componentProjection; - this.kafkaEventDispatcher = kafkaEventDispatcher; - this.queryManager = queryManager; - this.fetchMeta = fetchMeta; - } - - @Override - public IntegrityMetaComponent handle() throws MalformedPackageURLException { - IntegrityMetaComponent persistentIntegrityMetaComponent = queryManager.getIntegrityMetaComponent(componentProjection.purl().toString()); - if (persistentIntegrityMetaComponent == null) { - IntegrityMetaComponent integrityMetaComponent = queryManager.createIntegrityMetaComponent(createIntegrityMetaComponent(componentProjection.purl().toString())); - kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(componentProjection.componentUuid(), componentProjection.purl().canonicalize(), componentProjection.internal(), fetchMeta)); - return integrityMetaComponent; - } - if (persistentIntegrityMetaComponent.getStatus() == PROCESSED || persistentIntegrityMetaComponent.getStatus() == NOT_AVAILABLE) { - //only fetch the latest version because integrity data (hashes) is present - kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(componentProjection.componentUuid(), componentProjection.purl().canonicalize(), componentProjection.internal(), FetchMeta.FETCH_META_LATEST_VERSION)); - return persistentIntegrityMetaComponent; - } - if (persistentIntegrityMetaComponent.getStatus() == null || (persistentIntegrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS && Date.from(Instant.now()).getTime() - persistentIntegrityMetaComponent.getLastFetch().getTime() > TIME_SPAN)) { - persistentIntegrityMetaComponent.setLastFetch(Date.from(Instant.now())); - IntegrityMetaComponent updateIntegrityMetaComponent = queryManager.updateIntegrityMetaComponent(persistentIntegrityMetaComponent); - kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(componentProjection.componentUuid(), componentProjection.purl().canonicalize(), componentProjection.internal(), fetchMeta)); - return updateIntegrityMetaComponent; - } else { - kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(componentProjection.componentUuid(), componentProjection.purl().canonicalize(), componentProjection.internal(), fetchMeta)); - return persistentIntegrityMetaComponent; - } - } -} \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java b/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java deleted file mode 100644 index d629cc06b..000000000 --- a/src/main/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandler.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import com.github.packageurl.MalformedPackageURLException; -import com.github.packageurl.PackageURL; -import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; -import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; - -public class UnSupportedMetaHandler extends AbstractMetaHandler { - - public UnSupportedMetaHandler(ComponentProjection componentProjection, QueryManager queryManager, KafkaEventDispatcher kafkaEventDispatcher, FetchMeta fetchMeta) { - this.componentProjection = componentProjection; - this.kafkaEventDispatcher = kafkaEventDispatcher; - this.queryManager = queryManager; - this.fetchMeta = fetchMeta; - } - - @Override - public IntegrityMetaComponent handle() throws MalformedPackageURLException { - kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(null, new PackageURL(componentProjection.purlCoordinates()).canonicalize(), componentProjection.internal(), fetchMeta)); - return null; - } -} \ No newline at end of file diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessorInitializer.java b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessorInitializer.java index f96819fad..f580190fd 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/ProcessorInitializer.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/ProcessorInitializer.java @@ -37,7 +37,7 @@ public void contextInitialized(final ServletContextEvent event) { PROCESSOR_MANAGER.registerProcessor(VulnerabilityMirrorProcessor.PROCESSOR_NAME, KafkaTopics.NEW_VULNERABILITY, new VulnerabilityMirrorProcessor()); - PROCESSOR_MANAGER.registerProcessor(RepositoryMetaResultProcessor.PROCESSOR_NAME, + PROCESSOR_MANAGER.registerBatchProcessor(RepositoryMetaResultProcessor.PROCESSOR_NAME, KafkaTopics.REPO_META_ANALYSIS_RESULT, new RepositoryMetaResultProcessor()); PROCESSOR_MANAGER.registerBatchProcessor(EpssMirrorProcessor.PROCESSOR_NAME, KafkaTopics.NEW_EPSS, new EpssMirrorProcessor()); diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java index 5e6be8f2b..4d6b15364 100644 --- a/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java +++ b/src/main/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessor.java @@ -18,184 +18,255 @@ */ package org.dependencytrack.event.kafka.processor; +import alpine.Config; import alpine.common.logging.Logger; import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; -import org.apache.commons.lang3.StringUtils; +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.dependencytrack.common.ConfigKey; +import org.dependencytrack.event.kafka.processor.api.BatchProcessor; import org.dependencytrack.event.kafka.processor.api.Processor; import org.dependencytrack.event.kafka.processor.exception.ProcessingException; import org.dependencytrack.model.FetchStatus; import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.RepositoryMetaComponent; import org.dependencytrack.model.RepositoryType; -import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.persistence.jdbi.MetaComponentDao; import org.dependencytrack.proto.repometaanalysis.v1.AnalysisResult; -import org.dependencytrack.util.PersistenceUtil; +import org.dependencytrack.proto.repometaanalysis.v1.IntegrityMeta; +import org.dependencytrack.util.PurlUtil; -import javax.jdo.PersistenceManager; -import javax.jdo.Query; +import java.util.Comparator; import java.util.Date; -import java.util.Optional; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; -import static org.dependencytrack.event.kafka.componentmeta.IntegrityCheck.performIntegrityCheck; +import static org.apache.commons.lang3.StringUtils.trimToNull; +import static org.apache.commons.lang3.time.DateFormatUtils.ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT; +import static org.dependencytrack.event.kafka.processor.api.ProcessorUtils.withEnrichedMdc; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.useJdbiHandle; /** * A {@link Processor} responsible for processing result of component repository meta analyses. */ -public class RepositoryMetaResultProcessor implements Processor { +public class RepositoryMetaResultProcessor implements BatchProcessor { static final String PROCESSOR_NAME = "repo.meta.analysis.result"; private static final Logger LOGGER = Logger.getLogger(RepositoryMetaResultProcessor.class); @Override - public void process(final ConsumerRecord record) throws ProcessingException { - if (!isRecordValid(record)) { - return; - } - try (final var qm = new QueryManager()) { - synchronizeRepositoryMetadata(qm, record); - IntegrityMetaComponent integrityMetaComponent = synchronizeIntegrityMetadata(qm, record); - if (integrityMetaComponent != null) { - performIntegrityCheck(integrityMetaComponent, record.value(), qm); - } - } catch (Exception e) { - throw new ProcessingException(e); + public void process(final List> records) throws ProcessingException { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Consumed batch of %d record(s)".formatted(records.size())); } - } - private IntegrityMetaComponent synchronizeIntegrityMetadata(final QueryManager queryManager, final ConsumerRecord record) throws MalformedPackageURLException { - final AnalysisResult result = record.value(); - PackageURL purl = new PackageURL(result.getComponent().getPurl()); - if (result.hasIntegrityMeta()) { - return synchronizeIntegrityMetaResult(record, queryManager, purl); - } else { - LOGGER.debug("Incoming result for component with purl %s does not include component integrity info".formatted(purl)); - return null; + final List> validRecords = records.stream() + .filter(record -> withEnrichedMdc(record, () -> isRecordValid(record))) + .toList(); + if (validRecords.isEmpty()) { + LOGGER.warn("None of the %d consumed record(s) are valid; Skipping".formatted(records.size())); + return; } - } - private void synchronizeRepositoryMetadata(final QueryManager qm, final ConsumerRecord record) throws Exception { - final PersistenceManager pm = qm.getPersistenceManager(); - final AnalysisResult result = record.value(); - final var purl = new PackageURL(result.getComponent().getPurl()); - - // It is possible that the same meta info is reported for multiple components in parallel, - // causing unique constraint violations when attempting to insert into the REPOSITORY_META_COMPONENT table. - // In such cases, we can get away with simply retrying to SELECT+UPDATE or INSERT again. We'll attempt - // up to 3 times before giving up. - qm.runInRetryableTransaction(() -> { - final RepositoryMetaComponent repositoryMetaComponentResult = createRepositoryMetaResult(record, pm, purl); - if (repositoryMetaComponentResult != null) { - pm.makePersistent(repositoryMetaComponentResult); - } + final Map repoMetaComponentByIdentity = createRepoMetaComponents(records); + final Map integrityMetaComponentByPurl = createIntegrityMetaComponents(records); - return null; - }, PersistenceUtil::isUniqueConstraintViolation); + useJdbiHandle(handle -> { + final var dao = handle.attach(MetaComponentDao.class); + handle.useTransaction(ignored -> processRepositoryMetaComponents(dao, repoMetaComponentByIdentity)); + handle.useTransaction(ignored -> processIntegrityMetaComponents(dao, integrityMetaComponentByPurl)); + }); } - private RepositoryMetaComponent createRepositoryMetaResult(ConsumerRecord incomingAnalysisResultRecord, PersistenceManager pm, PackageURL purl) { - final AnalysisResult result = incomingAnalysisResultRecord.value(); - if (!result.hasLatestVersion()) { - return null; + private void processRepositoryMetaComponents( + final MetaComponentDao dao, + final Map metaComponentByIdentity + ) { + final var unprocessedMetaComponentByIdentity = new HashMap<>(metaComponentByIdentity); + + final List createdIdentities = + dao.createAllRepositoryMetaComponents(unprocessedMetaComponentByIdentity.values()); + createdIdentities.forEach(unprocessedMetaComponentByIdentity::remove); + + final List updatedIdentities = + dao.updateAllRepositoryMetaComponents(unprocessedMetaComponentByIdentity.values()); + updatedIdentities.forEach(unprocessedMetaComponentByIdentity::remove); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Processed repository meta component records: {created=%d, updated=%d, unchanged=%d}" + .formatted(createdIdentities.size(), updatedIdentities.size(), unprocessedMetaComponentByIdentity.size())); } + } - final Query query = pm.newQuery(RepositoryMetaComponent.class); - query.setFilter("repositoryType == :repositoryType && namespace == :namespace && name == :name"); - query.setParameters( - RepositoryType.resolve(purl), - purl.getNamespace(), - purl.getName() - ); + private void processIntegrityMetaComponents( + final MetaComponentDao dao, + final Map metaComponentByPurl + ) { + final var modifiedMetaComponentPurls = new HashSet(); + final var unprocessedMetaComponentByPurl = new HashMap<>(metaComponentByPurl); - RepositoryMetaComponent persistentRepoMetaComponent; - try { - persistentRepoMetaComponent = query.executeUnique(); - } finally { - query.closeAll(); + int numCreated = 0; + for (final String purl : dao.createAllIntegrityMetaComponents(unprocessedMetaComponentByPurl.values())) { + unprocessedMetaComponentByPurl.remove(purl); + modifiedMetaComponentPurls.add(purl); + numCreated++; } - if (persistentRepoMetaComponent == null) { - persistentRepoMetaComponent = new RepositoryMetaComponent(); + int numUpdated = 0; + if (!unprocessedMetaComponentByPurl.isEmpty()) { + for (final String purl : dao.updateAllIntegrityMetaComponents(unprocessedMetaComponentByPurl.values())) { + unprocessedMetaComponentByPurl.remove(purl); + modifiedMetaComponentPurls.add(purl); + numUpdated++; + } } - if (persistentRepoMetaComponent.getLastCheck() != null - && persistentRepoMetaComponent.getLastCheck().after(new Date(incomingAnalysisResultRecord.timestamp()))) { - LOGGER.warn(""" - Received repository meta information for %s that is older\s - than what's already in the database; Discarding - """.formatted(purl)); - return null; + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Processed integrity meta component records: {created=%d, updated=%d, unchanged=%d}" + .formatted(numCreated, numUpdated, unprocessedMetaComponentByPurl.size())); } - persistentRepoMetaComponent.setRepositoryType(RepositoryType.resolve(purl)); - persistentRepoMetaComponent.setNamespace(purl.getNamespace()); - persistentRepoMetaComponent.setName(purl.getName()); - if (result.hasLatestVersion()) { - persistentRepoMetaComponent.setLatestVersion(result.getLatestVersion()); + if (!Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_CHECK_ENABLED) + || (modifiedMetaComponentPurls.isEmpty())) { + return; } - if (result.hasPublished()) { - persistentRepoMetaComponent.setPublished(new Date(result.getPublished().getSeconds() * 1000)); + + // NB: Doing this here COULD, POTENTIALLY, EVENTUALLY cause the transaction to take too long. + // Depends on overall size of portfolio, the number of records in this batch, as well as the + // number of distinct PURLs in the batch. + // + // If this ever turns out to be problematic, we will need to switch to processing + // INTEGRITY_META_COMPONENT records and INTEGRITY_ANALYSIS one-by-one, rather than in batches. + // Persisting of meta component and analysis must be atomic. + // + // Reason being that if persisting of meta component record succeeds, but integrity analysis fails + // due to a transient error, we must ensure that retrying the processing of the Kafka record achieves + // the same result. If we do persisting and analysis in separate transactions, the analysis would not + // be repeated upon retry, since the meta component record is not registered as "modified" anymore. + final int modifiedAnalyses = dao.createOrUpdateIntegrityAnalysesForPurls(modifiedMetaComponentPurls); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Created or updated %d integrity analyses for %s" + .formatted(modifiedAnalyses, modifiedMetaComponentPurls)); } - persistentRepoMetaComponent.setLastCheck(new Date(incomingAnalysisResultRecord.timestamp())); - return persistentRepoMetaComponent; } - private IntegrityMetaComponent synchronizeIntegrityMetaResult(final ConsumerRecord incomingAnalysisResultRecord, QueryManager queryManager, PackageURL purl) { - final AnalysisResult result = incomingAnalysisResultRecord.value(); - IntegrityMetaComponent persistentIntegrityMetaComponent = queryManager.getIntegrityMetaComponent(purl.toString()); - if (persistentIntegrityMetaComponent != null && persistentIntegrityMetaComponent.getStatus() != null && persistentIntegrityMetaComponent.getStatus().equals(FetchStatus.PROCESSED)) { - LOGGER.warn(""" - Received hash information for %s that has already been processed; Discarding - """.formatted(purl)); - return persistentIntegrityMetaComponent; + @SuppressWarnings("RedundantStreamOptionalCall") // Calling .sorted before .collect is intentional. + private Map createRepoMetaComponents( + final List> records + ) { + final var identitiesSeen = new HashSet(); + return records.stream() + // We only store meta components for which the latest version was reported. + .filter(record -> record.value().hasLatestVersion()) + .map(record -> withEnrichedMdc(record, () -> createRepoMetaComponent(record))) + // Sort by lastFetch such that later timestamps appear first. + .sorted(Comparator.comparing(RepositoryMetaComponent::getLastCheck).reversed()) + // Keep only one (the latest) meta component for each repoType-namespace-name triplet. + .filter(metaComponent -> identitiesSeen.add(RepositoryMetaComponent.Identity.of(metaComponent))) + .collect(Collectors.toMap(RepositoryMetaComponent.Identity::of, Function.identity())); + } + + private static RepositoryMetaComponent createRepoMetaComponent(final ConsumerRecord record) { + final AnalysisResult result = record.value(); + final PackageURL purl = PurlUtil.silentPurl(result.getComponent().getPurl()); + + final var metaComponent = new RepositoryMetaComponent(); + metaComponent.setRepositoryType(RepositoryType.resolve(purl)); + metaComponent.setNamespace(purl.getNamespace()); + metaComponent.setName(purl.getName()); + metaComponent.setLatestVersion(result.getLatestVersion()); + if (result.hasPublished()) { + metaComponent.setPublished(new Date(Timestamps.toMillis(result.getPublished()))); } - if (persistentIntegrityMetaComponent == null) { - persistentIntegrityMetaComponent = new IntegrityMetaComponent(); + if (result.getFetchedAt() != Timestamp.getDefaultInstance()) { + metaComponent.setLastCheck(new Date(Timestamps.toMillis(result.getFetchedAt()))); + } else { + // For records sent by repository-meta-analyzer <=0.5.0. + final var recordTimestamp = new Date(record.timestamp()); + LOGGER.warn(""" + No fetched_at timestamp provided for repository metadata, \ + assuming record timestamp %s instead\ + """.formatted(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.format(recordTimestamp))); + metaComponent.setLastCheck(recordTimestamp); } + return metaComponent; + } - if (result.getIntegrityMeta().hasMd5() || result.getIntegrityMeta().hasSha1() || result.getIntegrityMeta().hasSha256() - || result.getIntegrityMeta().hasSha512() || result.getIntegrityMeta().hasCurrentVersionLastModified()) { - Optional.of(result.getIntegrityMeta().getMd5()).filter(StringUtils::isNotBlank).ifPresent(persistentIntegrityMetaComponent::setMd5); - Optional.of(result.getIntegrityMeta().getSha1()).filter(StringUtils::isNotBlank).ifPresent(persistentIntegrityMetaComponent::setSha1); - Optional.of(result.getIntegrityMeta().getSha256()).filter(StringUtils::isNotBlank).ifPresent(persistentIntegrityMetaComponent::setSha256); - Optional.of(result.getIntegrityMeta().getSha512()).filter(StringUtils::isNotBlank).ifPresent(persistentIntegrityMetaComponent::setSha512); - persistentIntegrityMetaComponent.setPurl(result.getComponent().getPurl()); - persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl()); - persistentIntegrityMetaComponent.setPublishedAt(result.getIntegrityMeta().hasCurrentVersionLastModified() ? new Date(result.getIntegrityMeta().getCurrentVersionLastModified().getSeconds() * 1000) : null); - persistentIntegrityMetaComponent.setStatus(FetchStatus.PROCESSED); + @SuppressWarnings("RedundantStreamOptionalCall") // Calling .sorted before .collect is intentional. + private static Map createIntegrityMetaComponents( + final List> records + ) { + final var purlsSeen = new HashSet(); + return records.stream() + // Not all ecosystems support retrieval of integrity data. + .filter(record -> record.value().hasIntegrityMeta()) + .map(record -> withEnrichedMdc(record, () -> createIntegrityMetaComponent(record))) + // Sort by lastFetch such that later timestamps appear first. + .sorted(Comparator.comparing(IntegrityMetaComponent::getLastFetch).reversed()) + // Only keep one (the latest) meta component for each PURL. + .filter(metaComponent -> purlsSeen.add(metaComponent.getPurl())) + .collect(Collectors.toMap(IntegrityMetaComponent::getPurl, Function.identity())); + } + + private static IntegrityMetaComponent createIntegrityMetaComponent(final ConsumerRecord record) { + final AnalysisResult result = record.value(); + final IntegrityMeta integrityMetaResult = result.getIntegrityMeta(); + + final var metaComponent = new IntegrityMetaComponent(); + metaComponent.setPurl(result.getComponent().getPurl()); + metaComponent.setRepositoryUrl(integrityMetaResult.getMetaSourceUrl()); + metaComponent.setMd5(trimToNull(integrityMetaResult.getMd5())); + metaComponent.setSha1(trimToNull(integrityMetaResult.getSha1())); + metaComponent.setSha256(trimToNull(integrityMetaResult.getSha256())); + metaComponent.setSha512(trimToNull(integrityMetaResult.getSha512())); + if (integrityMetaResult.hasCurrentVersionLastModified()) { + metaComponent.setPublishedAt(new Date(Timestamps.toMillis(integrityMetaResult.getCurrentVersionLastModified()))); + } + if (metaComponent.getMd5() != null + || metaComponent.getSha1() != null + || metaComponent.getSha256() != null + || metaComponent.getSha512() != null) { + metaComponent.setStatus(FetchStatus.PROCESSED); } else { - persistentIntegrityMetaComponent.setMd5(null); - persistentIntegrityMetaComponent.setSha256(null); - persistentIntegrityMetaComponent.setSha1(null); - persistentIntegrityMetaComponent.setSha512(null); - persistentIntegrityMetaComponent.setPurl(purl.toString()); - persistentIntegrityMetaComponent.setRepositoryUrl(result.getIntegrityMeta().getMetaSourceUrl()); - persistentIntegrityMetaComponent.setStatus(FetchStatus.NOT_AVAILABLE); + metaComponent.setStatus(FetchStatus.NOT_AVAILABLE); + } + if (integrityMetaResult.getFetchedAt() != Timestamp.getDefaultInstance()) { + metaComponent.setLastFetch(new Date(Timestamps.toMillis(integrityMetaResult.getFetchedAt()))); + } else { + // For records sent by repository-meta-analyzer <=0.5.0. + final var recordTimestamp = new Date(record.timestamp()); + LOGGER.warn(""" + No fetched_at timestamp provided for integrity metadata, \ + assuming record timestamp %s instead\ + """.formatted(ISO_8601_EXTENDED_DATETIME_TIME_ZONE_FORMAT.format(recordTimestamp))); + metaComponent.setLastFetch(recordTimestamp); } - return queryManager.updateIntegrityMetaComponent(persistentIntegrityMetaComponent); + + return metaComponent; } private static boolean isRecordValid(final ConsumerRecord record) { final AnalysisResult result = record.value(); if (!result.hasComponent()) { - LOGGER.warn(""" - Received repository meta information without component,\s - will not be able to correlate; Dropping - """); + LOGGER.warn("Component is missing; Dropping"); return false; } try { new PackageURL(result.getComponent().getPurl()); } catch (MalformedPackageURLException e) { - LOGGER.warn(""" - Received repository meta information with invalid PURL,\s - will not be able to correlate; Dropping - """, e); + LOGGER.warn("Invalid PURL; Dropping", e); return false; } + return true; } + } diff --git a/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorUtils.java b/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorUtils.java new file mode 100644 index 000000000..aa83c4ab8 --- /dev/null +++ b/src/main/java/org/dependencytrack/event/kafka/processor/api/ProcessorUtils.java @@ -0,0 +1,88 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.event.kafka.processor.api; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; +import com.google.protobuf.util.JsonFormat; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.slf4j.MDC; + +import java.util.UUID; +import java.util.concurrent.Callable; + +import static org.dependencytrack.common.MdcKeys.MDC_KAFKA_RECORD_KEY; +import static org.dependencytrack.common.MdcKeys.MDC_KAFKA_RECORD_OFFSET; +import static org.dependencytrack.common.MdcKeys.MDC_KAFKA_RECORD_PARTITION; +import static org.dependencytrack.common.MdcKeys.MDC_KAFKA_RECORD_TOPIC; + +/** + * @since 5.6.0 + */ +public final class ProcessorUtils { + + private ProcessorUtils() { + } + + /** + * Enriches the {@link MDC} with information about {@code record}, and executes {@code callable}. + * + * @param record The {@link ConsumerRecord} + * @param callable The {@link Callable} to execute + * @param Type of {@code callable}'s return value + * @return {@code callable}'s return value + */ + public static T withEnrichedMdc(final ConsumerRecord record, final Callable callable) { + try (var ignoredRecordTopic = MDC.putCloseable(MDC_KAFKA_RECORD_TOPIC, record.topic()); + var ignoredRecordPartition = MDC.putCloseable(MDC_KAFKA_RECORD_PARTITION, String.valueOf(record.partition())); + var ignoredRecordOffset = MDC.putCloseable(MDC_KAFKA_RECORD_OFFSET, String.valueOf(record.offset())); + var ignoredRecordKey = MDC.putCloseable(MDC_KAFKA_RECORD_KEY, printableKey(record))) { + return callable.call(); + } catch (final Exception e) { + if (e instanceof final RuntimeException re) { + throw re; + } + + throw new RuntimeException(e); + } + } + + private static String printableKey(final ConsumerRecord record) { + if (record.key() instanceof final String key) { + return key; + } else if (record.key() instanceof final UUID key) { + return key.toString(); + } else if (record.key() instanceof Number key) { + return String.valueOf(key); + } else if (record.key() instanceof Message key) { + try { + // JSON is easier to read than the default + // text-based representation of Proto messages. + return JsonFormat.printer() + .omittingInsignificantWhitespace() + .print(key); + } catch (InvalidProtocolBufferException e) { + return null; + } + } + + return null; + } + +} diff --git a/src/main/java/org/dependencytrack/model/RepositoryMetaComponent.java b/src/main/java/org/dependencytrack/model/RepositoryMetaComponent.java index 87ae3712b..fbcc02bf3 100644 --- a/src/main/java/org/dependencytrack/model/RepositoryMetaComponent.java +++ b/src/main/java/org/dependencytrack/model/RepositoryMetaComponent.java @@ -45,6 +45,21 @@ public class RepositoryMetaComponent implements Serializable { private static final long serialVersionUID = 4415041595179460918L; + /** + * @since 5.6.0 + */ + public record Identity( + RepositoryType repositoryType, + String namespace, + String name + ) { + + public static Identity of(final RepositoryMetaComponent metaComponent) { + return new Identity(metaComponent.getRepositoryType(), metaComponent.getNamespace(), metaComponent.getName()); + } + + } + @PrimaryKey @Persistent(valueStrategy = IdGeneratorStrategy.NATIVE) @JsonIgnore diff --git a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java index 5313517f9..ab5eafe5c 100644 --- a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java @@ -24,9 +24,6 @@ import alpine.resources.AlpineRequest; import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; -import jakarta.json.Json; -import jakarta.json.JsonArray; -import jakarta.json.JsonValue; import org.apache.commons.lang3.tuple.Pair; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; @@ -36,8 +33,10 @@ import org.dependencytrack.model.RepositoryType; import org.dependencytrack.model.sqlmapping.ComponentProjection; import org.dependencytrack.resources.v1.vo.DependencyGraphResponse; -import org.dependencytrack.tasks.IntegrityMetaInitializerTask; +import jakarta.json.Json; +import jakarta.json.JsonArray; +import jakarta.json.JsonValue; import javax.jdo.PersistenceManager; import javax.jdo.Query; import javax.jdo.Transaction; @@ -384,22 +383,6 @@ public PaginatedResult getComponentByHash(String hash) { return execute(query, params); } - /** - * Returns ComponentProjection for the purl. - * - * @param purl the purl of the component to retrieve - * @return associated ComponentProjection - */ - public IntegrityMetaInitializerTask.ComponentProjection getComponentByPurl(String purl) { - if (purl == null) { - return null; - } - final Query query = pm.newQuery(Component.class, "purl == :purl"); - query.setParameters(purl); - query.setResult("DISTINCT purlCoordinates, internal"); - return query.executeResultUnique(IntegrityMetaInitializerTask.ComponentProjection.class); - } - /** * Returns Components by their identity. * diff --git a/src/main/java/org/dependencytrack/persistence/IntegrityAnalysisQueryManager.java b/src/main/java/org/dependencytrack/persistence/IntegrityAnalysisQueryManager.java deleted file mode 100644 index 36aa12242..000000000 --- a/src/main/java/org/dependencytrack/persistence/IntegrityAnalysisQueryManager.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.persistence; - -import alpine.resources.AlpineRequest; -import org.dependencytrack.model.IntegrityAnalysis; - -import javax.jdo.PersistenceManager; -import javax.jdo.Query; -import java.util.UUID; - -public class IntegrityAnalysisQueryManager extends QueryManager implements IQueryManager { - - IntegrityAnalysisQueryManager(final PersistenceManager pm) { - super(pm); - } - - IntegrityAnalysisQueryManager(final PersistenceManager pm, final AlpineRequest request) { - super(pm, request); - } - - public IntegrityAnalysis getIntegrityAnalysisByComponentUuid(UUID uuid) { - final Query query = pm.newQuery(IntegrityAnalysis.class); - query.setFilter("component.uuid == :uuid"); - query.setParameters(uuid); - return query.executeUnique(); - } -} diff --git a/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java b/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java deleted file mode 100644 index 4522987e6..000000000 --- a/src/main/java/org/dependencytrack/persistence/IntegrityMetaQueryManager.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.persistence; - -import alpine.common.logging.Logger; -import alpine.resources.AlpineRequest; -import alpine.server.util.DbUtil; -import org.dependencytrack.model.FetchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; - -import javax.jdo.PersistenceManager; -import javax.jdo.Query; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.Timestamp; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Date; -import java.util.List; - -public class IntegrityMetaQueryManager extends QueryManager implements IQueryManager { - - private static final Logger LOGGER = Logger.getLogger(IntegrityMetaQueryManager.class); - - /** - * Constructs a new QueryManager. - * @param pm a PersistenceManager object - */ - IntegrityMetaQueryManager(final PersistenceManager pm) { - super(pm); - } - - /** - * Constructs a new QueryManager. - * @param pm a PersistenceManager object - * @param request an AlpineRequest object - */ - IntegrityMetaQueryManager(final PersistenceManager pm, final AlpineRequest request) { - super(pm, request); - } - - /** - * Returns a IntegrityMetaComponent object from the specified purl. - * - * @param purl the Package URL string of the component - * @return a IntegrityMetaComponent object, or null if not found - */ - public IntegrityMetaComponent getIntegrityMetaComponent(String purl) { - final Query query = pm.newQuery(IntegrityMetaComponent.class, "purl == :purl"); - query.setParameters(purl); - return query.executeUnique(); - } - - /** - * Updates a IntegrityMetaComponent record. - * - * @param transientIntegrityMetaComponent the IntegrityMetaComponent object to synchronize - * @return a synchronized IntegrityMetaComponent object - */ - public synchronized IntegrityMetaComponent updateIntegrityMetaComponent(final IntegrityMetaComponent transientIntegrityMetaComponent) { - final IntegrityMetaComponent integrityMeta = getIntegrityMetaComponent(transientIntegrityMetaComponent.getPurl()); - if (integrityMeta != null) { - integrityMeta.setMd5(transientIntegrityMetaComponent.getMd5()); - integrityMeta.setSha1(transientIntegrityMetaComponent.getSha1()); - integrityMeta.setSha256(transientIntegrityMetaComponent.getSha256()); - integrityMeta.setSha512(transientIntegrityMetaComponent.getSha512()); - integrityMeta.setPublishedAt(transientIntegrityMetaComponent.getPublishedAt()); - integrityMeta.setStatus(transientIntegrityMetaComponent.getStatus()); - integrityMeta.setLastFetch(Date.from(Instant.now())); - integrityMeta.setRepositoryUrl(transientIntegrityMetaComponent.getRepositoryUrl()); - return persist(integrityMeta); - } else { - LOGGER.debug("No record found in IntegrityMetaComponent for purl " + transientIntegrityMetaComponent.getPurl()); - return null; - } - } - - public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { - return persist(integrityMetaComponent); - } - - public void createIntegrityMetaHandlingConflict(IntegrityMetaComponent integrityMetaComponent) { - final String createQuery = """ - INSERT INTO "INTEGRITY_META_COMPONENT" ("PURL", "STATUS", "LAST_FETCH") - VALUES (?, ?, ?) - ON CONFLICT DO NOTHING - """; - Connection connection = null; - PreparedStatement preparedStatement = null; - try { - connection = (Connection) pm.getDataStoreConnection(); - preparedStatement = connection.prepareStatement(createQuery); - preparedStatement.setString(1, integrityMetaComponent.getPurl().toString()); - preparedStatement.setString(2, integrityMetaComponent.getStatus().toString()); - preparedStatement.setTimestamp(3, new java.sql.Timestamp(integrityMetaComponent.getLastFetch().getTime())); - preparedStatement.execute(); - } catch (Exception ex) { - LOGGER.error("Error in creating integrity meta component", ex); - throw new RuntimeException(ex); - } finally { - DbUtil.close(preparedStatement); - DbUtil.close(connection); - } - } - - /** - * Synchronizes IntegrityMetaComponent with purls from COMPONENT. This is part of initializer. - */ - public synchronized void synchronizeIntegrityMetaComponent() { - final String purlSyncQuery = """ - INSERT INTO "INTEGRITY_META_COMPONENT" ("PURL") - SELECT DISTINCT "PURL" - FROM "COMPONENT" - WHERE "PURL" IS NOT NULL - ON CONFLICT DO NOTHING - """; - Connection connection = null; - PreparedStatement preparedStatement = null; - try { - connection = (Connection) pm.getDataStoreConnection(); - preparedStatement = connection.prepareStatement(purlSyncQuery); - var purlCount = preparedStatement.executeUpdate(); - LOGGER.info("Number of component purls synchronized for integrity check : " + purlCount); - } catch (Exception ex) { - LOGGER.error("Error in synchronizing component purls for integrity meta.", ex); - throw new RuntimeException(ex); - } finally { - DbUtil.close(preparedStatement); - DbUtil.close(connection); - } - } - - /** - * Returns the count of records in IntegrityMetaComponent. - * - * @return the count of records - */ - public long getIntegrityMetaComponentCount() { - try (final Query query = pm.newQuery(IntegrityMetaComponent.class)) { - query.setResult("count(this)"); - return query.executeResultUnique(Long.class); - } catch (Exception e) { - LOGGER.error("Error in getting count of integrity meta.", e); - throw new RuntimeException(e); - } - } - - /** - * Returns the list of purls in IntegrityMetaComponent which are not yet processed. - * - * @return the list of purls - */ - public List fetchNextPurlsPage(long offset) { - try (final Query query = - pm.newQuery(IntegrityMetaComponent.class, "status == null || (status == :inProgress && lastFetch < :latest)")) { - query.setParameters(FetchStatus.IN_PROGRESS, Date.from(Instant.now().minus(1, ChronoUnit.HOURS))); - query.setRange(offset, offset + 5000); - query.setResult("id, purl"); - return List.copyOf(query.executeResultList(IntegrityMetaComponent.class)); - } catch (Exception e) { - LOGGER.error("Error in getting purls from integrity meta.", e); - throw new RuntimeException(e); - } - } - - /** - * Batch updates IntegrityMetaComponent records - */ - public void batchUpdateIntegrityMetaComponent(List purls) { - final String updateQuery = """ - UPDATE "INTEGRITY_META_COMPONENT" - SET "LAST_FETCH" = ?, "STATUS" = ? - WHERE "ID" = ? AND ("STATUS" IS NULL OR "STATUS" = 'IN_PROGRESS') - """; - Connection connection = null; - PreparedStatement preparedStatement = null; - try { - connection = (Connection) pm.getDataStoreConnection(); - preparedStatement = connection.prepareStatement(updateQuery); - for (var purlRecord : purls) { - preparedStatement.setTimestamp(1, new Timestamp(Date.from(Instant.now()).getTime())); - preparedStatement.setString(2, FetchStatus.IN_PROGRESS.toString()); - preparedStatement.setLong(3, purlRecord.getId()); - preparedStatement.addBatch(); - } - preparedStatement.executeBatch(); - } catch (Exception ex) { - LOGGER.error("Error in batch updating integrity meta.", ex); - throw new RuntimeException(ex); - } finally { - DbUtil.close(preparedStatement); - DbUtil.close(connection); - } - } -} diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java index 5a9592da1..5f8cb19d0 100644 --- a/src/main/java/org/dependencytrack/persistence/QueryManager.java +++ b/src/main/java/org/dependencytrack/persistence/QueryManager.java @@ -60,9 +60,7 @@ import org.dependencytrack.model.Epss; import org.dependencytrack.model.Finding; import org.dependencytrack.model.FindingAttribution; -import org.dependencytrack.model.IntegrityAnalysis; import org.dependencytrack.model.IntegrityMatchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.License; import org.dependencytrack.model.LicenseGroup; import org.dependencytrack.model.NotificationPublisher; @@ -100,7 +98,6 @@ import org.dependencytrack.proto.vulnanalysis.v1.ScannerResult; import org.dependencytrack.resources.v1.vo.AffectedProject; import org.dependencytrack.resources.v1.vo.DependencyGraphResponse; -import org.dependencytrack.tasks.IntegrityMetaInitializerTask; import javax.jdo.FetchPlan; import javax.jdo.PersistenceManager; @@ -160,8 +157,6 @@ public class QueryManager extends AlpineQueryManager { private VulnerabilityQueryManager vulnerabilityQueryManager; private VulnerableSoftwareQueryManager vulnerableSoftwareQueryManager; private WorkflowStateQueryManager workflowStateQueryManager; - private IntegrityMetaQueryManager integrityMetaQueryManager; - private IntegrityAnalysisQueryManager integrityAnalysisQueryManager; private TagQueryManager tagQueryManager; private EpssQueryManager epssQueryManager; @@ -463,20 +458,6 @@ private WorkflowStateQueryManager getWorkflowStateQueryManager() { return workflowStateQueryManager; } - private IntegrityMetaQueryManager getIntegrityMetaQueryManager() { - if (integrityMetaQueryManager == null) { - integrityMetaQueryManager = (request == null) ? new IntegrityMetaQueryManager(getPersistenceManager()) : new IntegrityMetaQueryManager(getPersistenceManager(), request); - } - return integrityMetaQueryManager; - } - - private IntegrityAnalysisQueryManager getIntegrityAnalysisQueryManager() { - if (integrityAnalysisQueryManager == null) { - integrityAnalysisQueryManager = (request == null) ? new IntegrityAnalysisQueryManager(getPersistenceManager()) : new IntegrityAnalysisQueryManager(getPersistenceManager(), request); - } - return integrityAnalysisQueryManager; - } - private void disableL2Cache() { pm.setProperty(PropertyNames.PROPERTY_CACHE_L2_TYPE, "none"); } @@ -698,10 +679,6 @@ public PaginatedResult getComponentByHash(String hash) { return getComponentQueryManager().getComponentByHash(hash); } - public IntegrityMetaInitializerTask.ComponentProjection getComponentByPurl(String purl) { - return getComponentQueryManager().getComponentByPurl(purl); - } - public PaginatedResult getComponents(ComponentIdentity identity) { return getComponentQueryManager().getComponents(identity); } @@ -1903,42 +1880,6 @@ public boolean hasWorkflowStepWithStatus(final UUID token, final WorkflowStep st return getWorkflowStateQueryManager().hasWorkflowStepWithStatus(token, step, status); } - public IntegrityMetaComponent getIntegrityMetaComponent(String purl) { - return getIntegrityMetaQueryManager().getIntegrityMetaComponent(purl); - } - - public IntegrityMetaComponent updateIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { - return getIntegrityMetaQueryManager().updateIntegrityMetaComponent(integrityMetaComponent); - } - - public void synchronizeIntegrityMetaComponent() { - getIntegrityMetaQueryManager().synchronizeIntegrityMetaComponent(); - } - - public long getIntegrityMetaComponentCount() { - return getIntegrityMetaQueryManager().getIntegrityMetaComponentCount(); - } - - public List fetchNextPurlsPage(long offset) { - return getIntegrityMetaQueryManager().fetchNextPurlsPage(offset); - } - - public void batchUpdateIntegrityMetaComponent(List purls) { - getIntegrityMetaQueryManager().batchUpdateIntegrityMetaComponent(purls); - } - - public IntegrityMetaComponent createIntegrityMetaComponent(IntegrityMetaComponent integrityMetaComponent) { - return getIntegrityMetaQueryManager().createIntegrityMetaComponent(integrityMetaComponent); - } - - public void createIntegrityMetaHandlingConflict(IntegrityMetaComponent integrityMetaComponent) { - getIntegrityMetaQueryManager().createIntegrityMetaHandlingConflict(integrityMetaComponent); - } - - public IntegrityAnalysis getIntegrityAnalysisByComponentUuid(UUID uuid) { - return getIntegrityAnalysisQueryManager().getIntegrityAnalysisByComponentUuid(uuid); - } - public ComponentMetaInformation getMetaInformation(UUID uuid) { Connection connection = null; diff --git a/src/main/java/org/dependencytrack/persistence/jdbi/MetaComponentDao.java b/src/main/java/org/dependencytrack/persistence/jdbi/MetaComponentDao.java new file mode 100644 index 000000000..8d4ae8d2b --- /dev/null +++ b/src/main/java/org/dependencytrack/persistence/jdbi/MetaComponentDao.java @@ -0,0 +1,230 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.persistence.jdbi; + +import org.dependencytrack.model.IntegrityMetaComponent; +import org.dependencytrack.model.RepositoryMetaComponent; +import org.jdbi.v3.sqlobject.config.RegisterConstructorMapper; +import org.jdbi.v3.sqlobject.customizer.Bind; +import org.jdbi.v3.sqlobject.customizer.BindBean; +import org.jdbi.v3.sqlobject.statement.GetGeneratedKeys; +import org.jdbi.v3.sqlobject.statement.SqlBatch; +import org.jdbi.v3.sqlobject.statement.SqlUpdate; + +import java.util.Collection; +import java.util.List; +import java.util.UUID; + +/** + * @since 5.6.0 + */ +public interface MetaComponentDao { + + @SqlBatch(""" + INSERT INTO "REPOSITORY_META_COMPONENT"( + "REPOSITORY_TYPE" + , "NAMESPACE" + , "NAME" + , "LATEST_VERSION" + , "PUBLISHED" + , "LAST_CHECK" + ) VALUES ( + :repositoryType + , :namespace + , :name + , :latestVersion + , :published + , :lastCheck + ) + ON CONFLICT DO NOTHING + RETURNING "REPOSITORY_TYPE" + , "NAMESPACE" + , "NAME" + """) + @GetGeneratedKeys + @RegisterConstructorMapper(RepositoryMetaComponent.Identity.class) + List createAllRepositoryMetaComponents( + @BindBean Collection metaComponents + ); + + @SqlBatch(""" + UPDATE "REPOSITORY_META_COMPONENT" + SET "LATEST_VERSION" = :latestVersion + , "PUBLISHED" = :published + , "LAST_CHECK" = :lastCheck + WHERE "REPOSITORY_TYPE" = :repositoryType + AND "NAMESPACE" = :namespace + AND "NAME" = :name + AND "LAST_CHECK" < :lastCheck + RETURNING "REPOSITORY_TYPE" + , "NAMESPACE" + , "NAME" + """) + @GetGeneratedKeys + @RegisterConstructorMapper(RepositoryMetaComponent.Identity.class) + List updateAllRepositoryMetaComponents( + @BindBean Collection metaComponents + ); + + @SqlBatch(""" + INSERT INTO "INTEGRITY_META_COMPONENT"( + "PURL" + , "REPOSITORY_URL" + , "PUBLISHED_AT" + , "MD5" + , "SHA1" + , "SHA256" + , "SHA512" + , "STATUS" + , "LAST_FETCH" + ) VALUES ( + :purl + , :repositoryUrl + , :publishedAt + , :md5 + , :sha1 + , :sha256 + , :sha512 + , :status + , :lastFetch + ) + ON CONFLICT DO NOTHING + RETURNING "PURL" + """) + @GetGeneratedKeys + List createAllIntegrityMetaComponents(@BindBean Collection metaComponents); + + @SqlBatch(""" + UPDATE "INTEGRITY_META_COMPONENT" + SET "REPOSITORY_URL" = :repositoryUrl + , "PUBLISHED_AT" = :publishedAt + , "MD5" = :md5 + , "SHA1" = :sha1 + , "SHA256" = :sha256 + , "SHA512" = :sha512 + , "STATUS" = :status + , "LAST_FETCH" = :lastFetch + WHERE "PURL" = :purl + AND "LAST_FETCH" < :lastFetch + RETURNING "PURL" + """) + @GetGeneratedKeys + List updateAllIntegrityMetaComponents(@BindBean Collection metaComponents); + + @SqlUpdate(""" + WITH "CTE_CHANGED_COMPONENT_HASH_MATCH_STATUS" AS ( + SELECT "C"."ID" AS "COMPONENT_ID" + , "MATCH_COMPONENT_HASH"("C"."MD5", "IMC"."MD5") AS "MD5_HASH_MATCH_STATUS" + , "MATCH_COMPONENT_HASH"("C"."SHA1", "IMC"."SHA1") AS "SHA1_HASH_MATCH_STATUS" + , "MATCH_COMPONENT_HASH"("C"."SHA_256", "IMC"."SHA256") AS "SHA256_HASH_MATCH_STATUS" + , "MATCH_COMPONENT_HASH"("C"."SHA3_512", "IMC"."SHA512") AS "SHA512_HASH_MATCH_STATUS" + FROM "COMPONENT" AS "C" + INNER JOIN "INTEGRITY_META_COMPONENT" AS "IMC" + ON "IMC"."PURL" = "C"."PURL" + LEFT JOIN "INTEGRITY_ANALYSIS" AS "IA" + ON "IA"."COMPONENT_ID" = "C"."ID" + WHERE "C"."PURL" = ANY(:purls) + AND ("MATCH_COMPONENT_HASH"("C"."MD5", "IMC"."MD5") IS DISTINCT FROM "IA"."MD5_HASH_MATCH_STATUS" + OR "MATCH_COMPONENT_HASH"("C"."SHA1", "IMC"."SHA1") IS DISTINCT FROM "IA"."SHA1_HASH_MATCH_STATUS" + OR "MATCH_COMPONENT_HASH"("C"."SHA_256", "IMC"."SHA256") IS DISTINCT FROM "IA"."SHA256_HASH_MATCH_STATUS" + OR "MATCH_COMPONENT_HASH"("C"."SHA_512", "IMC"."SHA512") IS DISTINCT FROM "IA"."SHA512_HASH_MATCH_STATUS") + ) + INSERT INTO "INTEGRITY_ANALYSIS"( + "COMPONENT_ID" + , "MD5_HASH_MATCH_STATUS" + , "SHA1_HASH_MATCH_STATUS" + , "SHA256_HASH_MATCH_STATUS" + , "SHA512_HASH_MATCH_STATUS" + , "INTEGRITY_CHECK_STATUS" + , "UPDATED_AT" + ) + SELECT "COMPONENT_ID" + , "MD5_HASH_MATCH_STATUS" + , "SHA1_HASH_MATCH_STATUS" + , "SHA256_HASH_MATCH_STATUS" + , "SHA512_HASH_MATCH_STATUS" + , "COMPONENT_INTEGRITY_CHECK_STATUS"( + "MD5_HASH_MATCH_STATUS" + , "SHA1_HASH_MATCH_STATUS" + , "SHA256_HASH_MATCH_STATUS" + , "SHA512_HASH_MATCH_STATUS" + ) AS "INTEGRITY_CHECK_STATUS" + , NOW() AS "UPDATED_AT" + FROM "CTE_CHANGED_COMPONENT_HASH_MATCH_STATUS" + ON CONFLICT ("COMPONENT_ID") DO UPDATE + SET "MD5_HASH_MATCH_STATUS" = EXCLUDED."MD5_HASH_MATCH_STATUS" + , "SHA1_HASH_MATCH_STATUS" = EXCLUDED."SHA1_HASH_MATCH_STATUS" + , "SHA256_HASH_MATCH_STATUS" = EXCLUDED."SHA256_HASH_MATCH_STATUS" + , "SHA512_HASH_MATCH_STATUS" = EXCLUDED."SHA512_HASH_MATCH_STATUS" + , "INTEGRITY_CHECK_STATUS" = EXCLUDED."INTEGRITY_CHECK_STATUS" + , "UPDATED_AT" = EXCLUDED."UPDATED_AT" + """) + int createOrUpdateIntegrityAnalysesForPurls(@Bind Collection purls); + + @SqlUpdate(""" + WITH "CTE_CHANGED_COMPONENT_HASH_MATCH_STATUS" AS ( + SELECT "C"."ID" AS "COMPONENT_ID" + , "MATCH_COMPONENT_HASH"("C"."MD5", "IMC"."MD5") AS "MD5_HASH_MATCH_STATUS" + , "MATCH_COMPONENT_HASH"("C"."SHA1", "IMC"."SHA1") AS "SHA1_HASH_MATCH_STATUS" + , "MATCH_COMPONENT_HASH"("C"."SHA_256", "IMC"."SHA256") AS "SHA256_HASH_MATCH_STATUS" + , "MATCH_COMPONENT_HASH"("C"."SHA3_512", "IMC"."SHA512") AS "SHA512_HASH_MATCH_STATUS" + FROM "COMPONENT" AS "C" + INNER JOIN "INTEGRITY_META_COMPONENT" AS "IMC" + ON "IMC"."PURL" = "C"."PURL" + LEFT JOIN "INTEGRITY_ANALYSIS" AS "IA" + ON "IA"."COMPONENT_ID" = "C"."ID" + WHERE "C"."UUID" = ANY((:uuids)::TEXT[]) + AND ("MATCH_COMPONENT_HASH"("C"."MD5", "IMC"."MD5") IS DISTINCT FROM "IA"."MD5_HASH_MATCH_STATUS" + OR "MATCH_COMPONENT_HASH"("C"."SHA1", "IMC"."SHA1") IS DISTINCT FROM "IA"."SHA1_HASH_MATCH_STATUS" + OR "MATCH_COMPONENT_HASH"("C"."SHA_256", "IMC"."SHA256") IS DISTINCT FROM "IA"."SHA256_HASH_MATCH_STATUS" + OR "MATCH_COMPONENT_HASH"("C"."SHA_512", "IMC"."SHA512") IS DISTINCT FROM "IA"."SHA512_HASH_MATCH_STATUS") + ) + INSERT INTO "INTEGRITY_ANALYSIS"( + "COMPONENT_ID" + , "MD5_HASH_MATCH_STATUS" + , "SHA1_HASH_MATCH_STATUS" + , "SHA256_HASH_MATCH_STATUS" + , "SHA512_HASH_MATCH_STATUS" + , "INTEGRITY_CHECK_STATUS" + , "UPDATED_AT" + ) + SELECT "COMPONENT_ID" + , "MD5_HASH_MATCH_STATUS" + , "SHA1_HASH_MATCH_STATUS" + , "SHA256_HASH_MATCH_STATUS" + , "SHA512_HASH_MATCH_STATUS" + , "COMPONENT_INTEGRITY_CHECK_STATUS"( + "MD5_HASH_MATCH_STATUS" + , "SHA1_HASH_MATCH_STATUS" + , "SHA256_HASH_MATCH_STATUS" + , "SHA512_HASH_MATCH_STATUS" + ) AS "INTEGRITY_CHECK_STATUS" + , NOW() AS "UPDATED_AT" + FROM "CTE_CHANGED_COMPONENT_HASH_MATCH_STATUS" + ON CONFLICT ("COMPONENT_ID") DO UPDATE + SET "MD5_HASH_MATCH_STATUS" = EXCLUDED."MD5_HASH_MATCH_STATUS" + , "SHA1_HASH_MATCH_STATUS" = EXCLUDED."SHA1_HASH_MATCH_STATUS" + , "SHA256_HASH_MATCH_STATUS" = EXCLUDED."SHA256_HASH_MATCH_STATUS" + , "SHA512_HASH_MATCH_STATUS" = EXCLUDED."SHA512_HASH_MATCH_STATUS" + , "INTEGRITY_CHECK_STATUS" = EXCLUDED."INTEGRITY_CHECK_STATUS" + , "UPDATED_AT" = EXCLUDED."UPDATED_AT" + """) + int createOrUpdateIntegrityAnalysesForComponents(@Bind Collection uuids); + +} diff --git a/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java b/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java index f0d50c721..e0939b75b 100644 --- a/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java +++ b/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java @@ -18,7 +18,7 @@ */ package org.dependencytrack.resources.v1; -import alpine.common.logging.Logger; +import alpine.Config; import alpine.event.framework.Event; import alpine.persistence.PaginatedResult; import alpine.server.auth.PermissionRequired; @@ -36,30 +36,15 @@ import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.security.SecurityRequirements; import io.swagger.v3.oas.annotations.tags.Tag; -import jakarta.validation.Validator; -import jakarta.ws.rs.Consumes; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.PUT; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.QueryParam; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; import org.dependencytrack.auth.Permissions; +import org.dependencytrack.common.ConfigKey; +import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent; import org.dependencytrack.event.InternalComponentIdentificationEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.event.kafka.componentmeta.ComponentProjection; -import org.dependencytrack.event.kafka.componentmeta.Handler; -import org.dependencytrack.event.kafka.componentmeta.HandlerFactory; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; -import org.dependencytrack.model.IntegrityAnalysis; -import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.License; import org.dependencytrack.model.Project; import org.dependencytrack.model.RepositoryMetaComponent; @@ -68,19 +53,30 @@ import org.dependencytrack.model.VulnerabilityScan; import org.dependencytrack.model.validation.ValidUuid; import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; +import org.dependencytrack.persistence.jdbi.MetaComponentDao; import org.dependencytrack.resources.v1.openapi.PaginatedApi; import org.dependencytrack.util.InternalComponentIdentifier; import org.dependencytrack.util.PurlUtil; +import jakarta.validation.Validator; +import jakarta.ws.rs.ClientErrorException; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.PUT; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.UUID; -import static org.dependencytrack.event.kafka.componentmeta.IntegrityCheck.calculateIntegrityResult; -import static org.dependencytrack.model.FetchStatus.NOT_AVAILABLE; -import static org.dependencytrack.model.FetchStatus.PROCESSED; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.useJdbiHandle; /** * JAX-RS resources for processing components. @@ -96,7 +92,6 @@ }) public class ComponentResource extends AlpineResource { - private static final Logger LOGGER = Logger.getLogger(ComponentResource.class); private final KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); @GET @@ -193,82 +188,6 @@ public Response getComponentByUuid( } } - @GET - @Path("/integritymetadata") - @Produces(MediaType.APPLICATION_JSON) - @Operation( - summary = """ - Provides the published date and hashes of the requested version of component, - as received from configured repositories for integrity analysis.""", - description = "

Requires permission VIEW_PORTFOLIO

" - ) - @ApiResponses(value = { - @ApiResponse( - responseCode = "200", - description = "Integrity metadata of the component", - content = @Content(schema = @Schema(implementation = IntegrityMetaComponent.class)) - ), - @ApiResponse(responseCode = "401", description = "Unauthorized"), - @ApiResponse(responseCode = "404", description = "The integrity meta information for the specified component cannot be found"), - @ApiResponse(responseCode = "400", description = "The package url being queried for is invalid") - }) - @PermissionRequired(Permissions.Constants.VIEW_PORTFOLIO) - public Response getIntegrityMetaComponent( - @Parameter(description = "The package url of the component", required = true) - @QueryParam("purl") String purl) { - try { - final PackageURL packageURL = new PackageURL(purl); - try (QueryManager qm = new QueryManager(getAlpineRequest())) { - final RepositoryType type = RepositoryType.resolve(packageURL); - if (RepositoryType.UNSUPPORTED == type) { - return Response.noContent().build(); - } - final IntegrityMetaComponent result = qm.getIntegrityMetaComponent(packageURL.toString()); - if (result == null) { - return Response.status(Response.Status.NOT_FOUND).entity("The integrity metadata for the specified component cannot be found.").build(); - } else { - //todo: future enhancement: provide pass-thru capability for component metadata not already present and being tracked - return Response.ok(result).build(); - } - } - } catch (MalformedPackageURLException e) { - return Response.status(Response.Status.BAD_REQUEST).build(); - } - } - - @GET - @Path("/{uuid}/integritycheckstatus") - @Produces(MediaType.APPLICATION_JSON) - @Operation( - summary = """ - Provides the integrity check status of component with provided UUID, - based on the configured repository for integrity analysis.""", - description = "

Requires permission VIEW_PORTFOLIO

" - ) - @ApiResponses(value = { - @ApiResponse( - responseCode = "200", - description = "Integrity metadata of the component", - content = @Content(schema = @Schema(implementation = IntegrityAnalysis.class)) - ), - @ApiResponse(responseCode = "401", description = "Unauthorized"), - @ApiResponse(responseCode = "404", description = "The integrity analysis information for the specified component cannot be found"), - }) - @PermissionRequired(Permissions.Constants.VIEW_PORTFOLIO) - public Response getIntegrityStatus( - @Parameter(description = "UUID of the component for which integrity status information is needed", schema = @Schema(type = "string", format = "uuid"), required = true) - @PathParam("uuid") @ValidUuid String uuid) { - try (QueryManager qm = new QueryManager(getAlpineRequest())) { - final IntegrityAnalysis result = qm.getIntegrityAnalysisByComponentUuid(UUID.fromString(uuid)); - if (result == null) { - return Response.status(Response.Status.NOT_FOUND).entity("The integrity status for the specified component cannot be found.").build(); - } else { - //todo: future enhancement: provide pass-thru capability for component metadata not already present and being tracked - return Response.ok(result).build(); - } - } - } - @GET @Path("/identity") @Produces(MediaType.APPLICATION_JSON) @@ -409,79 +328,82 @@ public Response createComponent(@Parameter(description = "The UUID of the projec validator.validateProperty(jsonComponent, "sha3_512") ); - try (QueryManager qm = new QueryManager()) { - Component parent = null; - if (jsonComponent.getParent() != null && jsonComponent.getParent().getUuid() != null) { - parent = qm.getObjectByUuid(Component.class, jsonComponent.getParent().getUuid()); - } - final Project project = qm.getObjectByUuid(Project.class, uuid); - if (project == null) { - return Response.status(Response.Status.NOT_FOUND).entity("The project could not be found.").build(); - } - if (!qm.hasAccess(super.getPrincipal(), project)) { - return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified project is forbidden").build(); - } - final License resolvedLicense = qm.getLicense(jsonComponent.getLicense()); - Component component = new Component(); - component.setProject(project); - component.setAuthor(StringUtils.trimToNull(jsonComponent.getAuthor())); - component.setPublisher(StringUtils.trimToNull(jsonComponent.getPublisher())); - component.setName(StringUtils.trimToNull(jsonComponent.getName())); - component.setVersion(StringUtils.trimToNull(jsonComponent.getVersion())); - component.setGroup(StringUtils.trimToNull(jsonComponent.getGroup())); - component.setDescription(StringUtils.trimToNull(jsonComponent.getDescription())); - component.setFilename(StringUtils.trimToNull(jsonComponent.getFilename())); - component.setClassifier(jsonComponent.getClassifier()); - component.setPurl(jsonComponent.getPurl()); - component.setPurlCoordinates(PurlUtil.silentPurlCoordinatesOnly(jsonComponent.getPurl())); - component.setInternal(new InternalComponentIdentifier().isInternal(component)); - component.setCpe(StringUtils.trimToNull(jsonComponent.getCpe())); - component.setSwidTagId(StringUtils.trimToNull(jsonComponent.getSwidTagId())); - component.setCopyright(StringUtils.trimToNull(jsonComponent.getCopyright())); - component.setMd5(StringUtils.trimToNull(jsonComponent.getMd5())); - component.setSha1(StringUtils.trimToNull(jsonComponent.getSha1())); - component.setSha256(StringUtils.trimToNull(jsonComponent.getSha256())); - component.setSha384(StringUtils.trimToNull(jsonComponent.getSha384())); - component.setSha512(StringUtils.trimToNull(jsonComponent.getSha512())); - component.setSha3_256(StringUtils.trimToNull(jsonComponent.getSha3_256())); - component.setSha3_384(StringUtils.trimToNull(jsonComponent.getSha3_384())); - component.setSha3_512(StringUtils.trimToNull(jsonComponent.getSha3_512())); - if (resolvedLicense != null) { - component.setLicense(null); - component.setLicenseExpression(null); - component.setLicenseUrl(StringUtils.trimToNull(jsonComponent.getLicenseUrl())); - component.setResolvedLicense(resolvedLicense); - } else if (StringUtils.isNotBlank(jsonComponent.getLicense())) { - component.setLicense(StringUtils.trim(jsonComponent.getLicense())); - component.setLicenseExpression(null); - component.setLicenseUrl(StringUtils.trimToNull(jsonComponent.getLicenseUrl())); - component.setResolvedLicense(null); - } else if (StringUtils.isNotBlank(jsonComponent.getLicenseExpression())) { - component.setLicense(null); - component.setLicenseExpression(StringUtils.trim(jsonComponent.getLicenseExpression())); - component.setLicenseUrl(null); - component.setResolvedLicense(null); - } - component.setParent(parent); - component.setNotes(StringUtils.trimToNull(jsonComponent.getNotes())); - - component = qm.createComponent(component, true); - ComponentProjection componentProjection = - new ComponentProjection(component.getUuid(), component.getPurlCoordinates().toString(), - component.isInternal(), component.getPurl()); - try { - Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION); - IntegrityMetaComponent integrityMetaComponent = repoMetaHandler.handle(); - if (integrityMetaComponent != null && (integrityMetaComponent.getStatus() == PROCESSED || integrityMetaComponent.getStatus() == NOT_AVAILABLE)) { - calculateIntegrityResult(integrityMetaComponent, component, qm); + try (final var qm = new QueryManager()) { + final Component createdComponent = qm.callInTransaction(() -> { + Component parent = null; + if (jsonComponent.getParent() != null && jsonComponent.getParent().getUuid() != null) { + parent = qm.getObjectByUuid(Component.class, jsonComponent.getParent().getUuid()); } - } catch (MalformedPackageURLException ex) { - LOGGER.warn("Unable to process package url %s".formatted(componentProjection.purl())); + final Project project = qm.getObjectByUuid(Project.class, uuid); + if (project == null) { + throw new ClientErrorException(Response + .status(Response.Status.NOT_FOUND) + .entity("The project could not be found.") + .build()); + } + if (!qm.hasAccess(super.getPrincipal(), project)) { + throw new ClientErrorException(Response + .status(Response.Status.FORBIDDEN) + .entity("Access to the specified project is forbidden") + .build()); + } + final License resolvedLicense = qm.getLicense(jsonComponent.getLicense()); + Component component = new Component(); + component.setProject(project); + component.setAuthor(StringUtils.trimToNull(jsonComponent.getAuthor())); + component.setPublisher(StringUtils.trimToNull(jsonComponent.getPublisher())); + component.setName(StringUtils.trimToNull(jsonComponent.getName())); + component.setVersion(StringUtils.trimToNull(jsonComponent.getVersion())); + component.setGroup(StringUtils.trimToNull(jsonComponent.getGroup())); + component.setDescription(StringUtils.trimToNull(jsonComponent.getDescription())); + component.setFilename(StringUtils.trimToNull(jsonComponent.getFilename())); + component.setClassifier(jsonComponent.getClassifier()); + component.setPurl(jsonComponent.getPurl()); + component.setPurlCoordinates(PurlUtil.silentPurlCoordinatesOnly(jsonComponent.getPurl())); + component.setInternal(new InternalComponentIdentifier().isInternal(component)); + component.setCpe(StringUtils.trimToNull(jsonComponent.getCpe())); + component.setSwidTagId(StringUtils.trimToNull(jsonComponent.getSwidTagId())); + component.setCopyright(StringUtils.trimToNull(jsonComponent.getCopyright())); + component.setMd5(StringUtils.trimToNull(jsonComponent.getMd5())); + component.setSha1(StringUtils.trimToNull(jsonComponent.getSha1())); + component.setSha256(StringUtils.trimToNull(jsonComponent.getSha256())); + component.setSha384(StringUtils.trimToNull(jsonComponent.getSha384())); + component.setSha512(StringUtils.trimToNull(jsonComponent.getSha512())); + component.setSha3_256(StringUtils.trimToNull(jsonComponent.getSha3_256())); + component.setSha3_384(StringUtils.trimToNull(jsonComponent.getSha3_384())); + component.setSha3_512(StringUtils.trimToNull(jsonComponent.getSha3_512())); + if (resolvedLicense != null) { + component.setLicense(null); + component.setLicenseExpression(null); + component.setLicenseUrl(StringUtils.trimToNull(jsonComponent.getLicenseUrl())); + component.setResolvedLicense(resolvedLicense); + } else if (StringUtils.isNotBlank(jsonComponent.getLicense())) { + component.setLicense(StringUtils.trim(jsonComponent.getLicense())); + component.setLicenseExpression(null); + component.setLicenseUrl(StringUtils.trimToNull(jsonComponent.getLicenseUrl())); + component.setResolvedLicense(null); + } else if (StringUtils.isNotBlank(jsonComponent.getLicenseExpression())) { + component.setLicense(null); + component.setLicenseExpression(StringUtils.trim(jsonComponent.getLicenseExpression())); + component.setLicenseUrl(null); + component.setResolvedLicense(null); + } + component.setParent(parent); + component.setNotes(StringUtils.trimToNull(jsonComponent.getNotes())); + + return qm.createComponent(component, true); + }); + + if (Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_CHECK_ENABLED)) { + useJdbiHandle(handle -> handle.attach(MetaComponentDao.class) + .createOrUpdateIntegrityAnalysesForComponents(List.of(createdComponent.getUuid()))); } - final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), component, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, true); - qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, component.getUuid(), vulnAnalysisEvent.token().toString(), 1); + + kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(createdComponent.getPurl(), createdComponent.isInternal())); + final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), createdComponent, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, true); + qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, createdComponent.getUuid(), vulnAnalysisEvent.token().toString(), 1); kafkaEventDispatcher.dispatchEvent(vulnAnalysisEvent); - return Response.status(Response.Status.CREATED).entity(component).build(); + return Response.status(Response.Status.CREATED).entity(createdComponent).build(); } } @@ -526,11 +448,20 @@ public Response updateComponent(Component jsonComponent) { validator.validateProperty(jsonComponent, "sha3_256"), validator.validateProperty(jsonComponent, "sha3_512") ); - try (QueryManager qm = new QueryManager()) { - Component component = qm.getObjectByUuid(Component.class, jsonComponent.getUuid()); - if (component != null) { + try (final var qm = new QueryManager()) { + final Component updatedComponent = qm.callInTransaction(() -> { + Component component = qm.getObjectByUuid(Component.class, jsonComponent.getUuid()); + if (component == null) { + throw new ClientErrorException(Response + .status(Response.Status.NOT_FOUND) + .entity("The UUID of the component could not be found.") + .build()); + } if (!qm.hasAccess(super.getPrincipal(), component.getProject())) { - return Response.status(Response.Status.FORBIDDEN).entity("Access to the specified component is forbidden").build(); + throw new ClientErrorException(Response + .status(Response.Status.FORBIDDEN) + .entity("Access to the specified component is forbidden") + .build()); } // Name cannot be empty or null - prevent it final String name = StringUtils.trimToNull(jsonComponent.getName()); @@ -584,27 +515,19 @@ public Response updateComponent(Component jsonComponent) { } component.setNotes(StringUtils.trimToNull(jsonComponent.getNotes())); - component = qm.updateComponent(component, true); - ComponentProjection componentProjection = - new ComponentProjection(component.getUuid(), component.getPurlCoordinates().toString(), - component.isInternal(), component.getPurl()); - try { + return qm.updateComponent(component, true); + }); - Handler repoMetaHandler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION); - IntegrityMetaComponent integrityMetaComponent = repoMetaHandler.handle(); - if (integrityMetaComponent != null && (integrityMetaComponent.getStatus() == PROCESSED || integrityMetaComponent.getStatus() == NOT_AVAILABLE)) { - calculateIntegrityResult(integrityMetaComponent, component, qm); - } - } catch (MalformedPackageURLException ex) { - LOGGER.warn("Unable to determine package url type for this purl %s".formatted(component.getPurl().getType()), ex); - } - final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), component, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, false); - qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, component.getUuid(), vulnAnalysisEvent.token().toString(), 1); - kafkaEventDispatcher.dispatchEvent(vulnAnalysisEvent); - return Response.ok(component).build(); - } else { - return Response.status(Response.Status.NOT_FOUND).entity("The UUID of the component could not be found.").build(); + if (Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_CHECK_ENABLED)) { + useJdbiHandle(handle -> handle.attach(MetaComponentDao.class) + .createOrUpdateIntegrityAnalysesForComponents(List.of(updatedComponent.getUuid()))); } + + kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(updatedComponent.getPurl(), updatedComponent.isInternal())); + final var vulnAnalysisEvent = new ComponentVulnerabilityAnalysisEvent(UUID.randomUUID(), updatedComponent, VulnerabilityAnalysisLevel.MANUAL_ANALYSIS, false); + qm.createVulnerabilityScan(VulnerabilityScan.TargetType.COMPONENT, updatedComponent.getUuid(), vulnAnalysisEvent.token().toString(), 1); + kafkaEventDispatcher.dispatchEvent(vulnAnalysisEvent); + return Response.ok(updatedComponent).build(); } } diff --git a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java index 287a5466f..a1566ca62 100644 --- a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java +++ b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTask.java @@ -22,7 +22,6 @@ import alpine.common.logging.Logger; import alpine.event.framework.ChainableEvent; import alpine.event.framework.Event; -import alpine.event.framework.EventService; import alpine.event.framework.Subscriber; import alpine.notification.Notification; import alpine.notification.NotificationLevel; @@ -38,15 +37,11 @@ import org.dependencytrack.event.BomUploadEvent; import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; import org.dependencytrack.event.ComponentVulnerabilityAnalysisEvent; -import org.dependencytrack.event.IntegrityAnalysisEvent; import org.dependencytrack.event.ProjectMetricsUpdateEvent; import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.event.kafka.componentmeta.AbstractMetaHandler; import org.dependencytrack.model.Bom; import org.dependencytrack.model.Component; import org.dependencytrack.model.ComponentIdentity; -import org.dependencytrack.model.FetchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; import org.dependencytrack.model.License; import org.dependencytrack.model.Project; import org.dependencytrack.model.ProjectMetadata; @@ -62,6 +57,7 @@ import org.dependencytrack.notification.vo.BomConsumedOrProcessed; import org.dependencytrack.notification.vo.BomProcessingFailed; import org.dependencytrack.persistence.QueryManager; +import org.dependencytrack.persistence.jdbi.MetaComponentDao; import org.dependencytrack.util.InternalComponentIdentifier; import org.dependencytrack.util.WaitingLockConfiguration; import org.json.JSONArray; @@ -89,6 +85,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; +import java.util.stream.Collectors; import static org.apache.commons.lang3.StringUtils.isNotBlank; import static org.apache.commons.lang3.StringUtils.trim; @@ -105,16 +102,13 @@ import static org.dependencytrack.common.MdcKeys.MDC_PROJECT_NAME; import static org.dependencytrack.common.MdcKeys.MDC_PROJECT_UUID; import static org.dependencytrack.common.MdcKeys.MDC_PROJECT_VERSION; -import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK; -import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.TIME_SPAN; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertComponents; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertDependencyGraph; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertServices; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertToProject; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.convertToProjectMetadata; import static org.dependencytrack.parser.cyclonedx.util.ModelConverter.flatten; -import static org.dependencytrack.proto.repometaanalysis.v1.FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION; -import static org.dependencytrack.proto.repometaanalysis.v1.FetchMeta.FETCH_META_LATEST_VERSION; +import static org.dependencytrack.persistence.jdbi.JdbiFactory.inJdbiTransaction; import static org.dependencytrack.util.LockProvider.executeWithLockWaiting; import static org.dependencytrack.util.PersistenceUtil.applyIfChanged; import static org.dependencytrack.util.PersistenceUtil.assertPersistent; @@ -229,6 +223,7 @@ private void processEvent(final Context ctx, final BomUploadEvent event) { dispatchBomProcessedNotification(ctx); } + maybePerformIntegrityAnalysis(processedBom.components()); final List vulnAnalysisEvents = createVulnAnalysisEvents(ctx, processedBom.components()); final List repoMetaAnalysisEvents = createRepoMetaAnalysisEvents(processedBom.components()); @@ -994,6 +989,24 @@ private void dispatchBomProcessingFailedNotification(final Context ctx, final Th .subject(new BomProcessingFailed(ctx.token, ctx.project, /* bom */ "(Omitted)", throwable.getMessage(), ctx.bomFormat, ctx.bomSpecVersion))); } + private static void maybePerformIntegrityAnalysis(final Collection components) { + if (!Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_CHECK_ENABLED)) { + LOGGER.debug("Not performing integrity check because %s is disabled" + .formatted(ConfigKey.INTEGRITY_CHECK_ENABLED.getPropertyName())); + return; + } + + final Set newComponentUuids = components.stream() + .filter(Component::isNew) + .map(Component::getUuid) + .collect(Collectors.toSet()); + final int createdAnalyses = inJdbiTransaction(handle -> handle.attach(MetaComponentDao.class) + .createOrUpdateIntegrityAnalysesForComponents(newComponentUuids)); + if (createdAnalyses > 0) { + LOGGER.debug("Created integrity analyses for %d component(s)".formatted(createdAnalyses)); + } + } + private static List createVulnAnalysisEvents( final Context ctx, final Collection components @@ -1009,70 +1022,12 @@ private static List createVulnAnalysisEvent } private static List createRepoMetaAnalysisEvents(final Collection components) { - final var events = new ArrayList(components.size()); - // TODO: This should be more efficient (https://github.com/DependencyTrack/hyades/issues/1306) - - try (final var qm = new QueryManager()) { - qm.getPersistenceManager().setProperty(PROPERTY_PERSISTENCE_BY_REACHABILITY_AT_COMMIT, "false"); - qm.getPersistenceManager().setProperty(PROPERTY_RETAIN_VALUES, "true"); - - for (final Component component : components) { - if (component.getPurl() == null) { - continue; - } - - if (!SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(component.getPurl().getType())) { - events.add(new ComponentRepositoryMetaAnalysisEvent( - /* componentUuid */ null, - component.getPurlCoordinates().toString(), - component.isInternal(), - FETCH_META_LATEST_VERSION - )); - continue; - } - - final boolean shouldFetchIntegrityData = qm.callInTransaction(() -> prepareIntegrityMetaComponent(qm, component)); - if (shouldFetchIntegrityData) { - events.add(new ComponentRepositoryMetaAnalysisEvent( - component.getUuid(), - component.getPurl().toString(), - component.isInternal(), - FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION - )); - } else { - // If integrity metadata was fetched recently, we don't want to fetch it again - // as it's unlikely to change frequently. Fall back to fetching only the latest - // version information. - events.add(new ComponentRepositoryMetaAnalysisEvent( - /* componentUuid */ null, - component.getPurlCoordinates().toString(), - component.isInternal(), - FETCH_META_LATEST_VERSION - )); - } - } - } - - return events; - } - - private static boolean prepareIntegrityMetaComponent(final QueryManager qm, final Component component) { - final IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(component.getPurlCoordinates().toString()); - if (integrityMetaComponent == null) { - qm.createIntegrityMetaHandlingConflict(AbstractMetaHandler.createIntegrityMetaComponent(component.getPurlCoordinates().toString())); - return true; - } else if (integrityMetaComponent.getStatus() == null - || (integrityMetaComponent.getStatus() == FetchStatus.IN_PROGRESS - && (Date.from(Instant.now()).getTime() - integrityMetaComponent.getLastFetch().getTime()) > TIME_SPAN)) { - integrityMetaComponent.setLastFetch(Date.from(Instant.now())); - return true; - } else if (integrityMetaComponent.getStatus() == FetchStatus.PROCESSED || integrityMetaComponent.getStatus() == FetchStatus.NOT_AVAILABLE) { - qm.getPersistenceManager().makeTransient(integrityMetaComponent); - EventService.getInstance().publish(new IntegrityAnalysisEvent(component.getUuid(), integrityMetaComponent)); - return false; - } - //don't send event because integrity metadata would be sent recently and don't want to send again - return false; + return components.stream() + .filter(component -> component.getPurl() != null) + .map(component -> Map.entry(component.getPurl(), component.isInternal())) + .distinct() + .map(entry -> new ComponentRepositoryMetaAnalysisEvent(/* purl */ entry.getKey(), /* internal */ entry.getValue())) + .toList(); } private static WaitingLockConfiguration createLockConfiguration(final Context ctx) { diff --git a/src/main/java/org/dependencytrack/tasks/IntegrityAnalysisTask.java b/src/main/java/org/dependencytrack/tasks/IntegrityAnalysisTask.java deleted file mode 100644 index 8a7a93fe8..000000000 --- a/src/main/java/org/dependencytrack/tasks/IntegrityAnalysisTask.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.tasks; - -import alpine.Config; -import alpine.common.logging.Logger; -import alpine.event.framework.Event; -import alpine.event.framework.Subscriber; -import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.event.IntegrityAnalysisEvent; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.persistence.QueryManager; - -import java.util.UUID; - -import static org.dependencytrack.event.kafka.componentmeta.IntegrityCheck.calculateIntegrityResult; - -public class IntegrityAnalysisTask implements Subscriber { - - private static final Logger LOGGER = Logger.getLogger(IntegrityAnalysisTask.class); - - @Override - public void inform(final Event e) { - if (e instanceof final IntegrityAnalysisEvent event) { - if (!Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_CHECK_ENABLED)) { - return; - } - LOGGER.debug("Performing integrity analysis for component: " + event.getUuid()); - if(event.getUuid() == null) { - return; - } - try (final var qm = new QueryManager()) { - UUID uuid = event.getUuid(); - IntegrityMetaComponent integrityMetaComponent = event.getIntegrityMetaComponent(); - Component component = qm.getObjectByUuid(Component.class, uuid); - calculateIntegrityResult(integrityMetaComponent, component, qm); - } - } - } -} diff --git a/src/main/java/org/dependencytrack/tasks/IntegrityMetaInitializerTask.java b/src/main/java/org/dependencytrack/tasks/IntegrityMetaInitializerTask.java deleted file mode 100644 index a8ebf015f..000000000 --- a/src/main/java/org/dependencytrack/tasks/IntegrityMetaInitializerTask.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.tasks; - -import alpine.Config; -import alpine.common.logging.Logger; -import alpine.event.framework.Event; -import alpine.event.framework.Subscriber; -import com.github.packageurl.MalformedPackageURLException; -import com.github.packageurl.PackageURL; -import org.dependencytrack.common.ConfigKey; -import org.dependencytrack.event.ComponentRepositoryMetaAnalysisEvent; -import org.dependencytrack.event.IntegrityMetaInitializerEvent; -import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.persistence.QueryManager; - -import java.util.List; - -import static org.dependencytrack.event.kafka.componentmeta.RepoMetaConstants.SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK; -import static org.dependencytrack.proto.repometaanalysis.v1.FetchMeta.FETCH_META_INTEGRITY_DATA; - -public class IntegrityMetaInitializerTask implements Subscriber { - - private static final Logger LOGGER = Logger.getLogger(IntegrityMetaInitializerTask.class); - - private final KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); - - public void inform(final Event e) { - if (e instanceof IntegrityMetaInitializerEvent) { - if (!Config.getInstance().getPropertyAsBoolean(ConfigKey.INTEGRITY_INITIALIZER_ENABLED)) { - LOGGER.debug("Integrity initializer is disabled"); - return; - } - try (final var qm = new QueryManager()) { - batchProcessPurls(qm); - } - } - } - - private void batchProcessPurls(QueryManager qm) { - long offset = 0; - List integrityMetaPurls = qm.fetchNextPurlsPage(offset); - while (!integrityMetaPurls.isEmpty()) { - dispatchPurls(qm, integrityMetaPurls); - qm.batchUpdateIntegrityMetaComponent(integrityMetaPurls); - offset += integrityMetaPurls.size(); - integrityMetaPurls = qm.fetchNextPurlsPage(offset); - } - } - - private void dispatchPurls(QueryManager qm, List integrityMetaComponents) { - for (final IntegrityMetaComponent integrityMetaComponent : integrityMetaComponents) { - try { - PackageURL purl = new PackageURL(integrityMetaComponent.getPurl()); - //dispatch for integrity metadata only if purl type is supported - if (SUPPORTED_PACKAGE_URLS_FOR_INTEGRITY_CHECK.contains(purl.getType())) { - IntegrityMetaInitializerTask.ComponentProjection componentProjection = qm.getComponentByPurl(integrityMetaComponent.getPurl()); - if (componentProjection == null) { - LOGGER.debug("No component with PURL %s exists (anymore); Skipping".formatted(integrityMetaComponent.getPurl())); - continue; - } - LOGGER.debug("Dispatching purl for integrity metadata: " + integrityMetaComponent.getPurl()); - //Initializer will not trigger Integrity Check on component so component uuid is not required - kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(null, integrityMetaComponent.getPurl(), componentProjection.internal(), FETCH_META_INTEGRITY_DATA)); - } - } catch (MalformedPackageURLException packageURLException) { - LOGGER.warn("Initializer cannot dispatch for integrity because purl cannot be parse: " + integrityMetaComponent.getPurl()); - //skip malformed url - } - } - } - - public record ComponentProjection(String purlCoordinates, Boolean internal) { - } -} diff --git a/src/main/java/org/dependencytrack/tasks/LockName.java b/src/main/java/org/dependencytrack/tasks/LockName.java index e94a215b1..46ee28992 100644 --- a/src/main/java/org/dependencytrack/tasks/LockName.java +++ b/src/main/java/org/dependencytrack/tasks/LockName.java @@ -27,6 +27,5 @@ public enum LockName { WORKFLOW_STEP_CLEANUP_TASK_LOCK, PORTFOLIO_REPO_META_ANALYSIS_TASK_LOCK, PORTFOLIO_VULN_ANALYSIS_TASK_LOCK, - INTEGRITY_META_INITIALIZER_LOCK, VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK } diff --git a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java index 68e6f0399..601d2155a 100644 --- a/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java +++ b/src/main/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTask.java @@ -21,6 +21,7 @@ import alpine.common.logging.Logger; import alpine.event.framework.Event; import alpine.event.framework.Subscriber; +import com.github.packageurl.PackageURL; import net.javacrumbs.shedlock.core.LockConfiguration; import net.javacrumbs.shedlock.core.LockExtender; import net.javacrumbs.shedlock.core.LockingTaskExecutor; @@ -31,8 +32,8 @@ import org.dependencytrack.model.Component; import org.dependencytrack.model.Project; import org.dependencytrack.persistence.QueryManager; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; import org.dependencytrack.util.LockProvider; +import org.dependencytrack.util.PurlUtil; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -50,7 +51,7 @@ * analysis. *

* As repository metadata analysis is purely based on PURLs, and does not (currently) consider PURL qualifiers, - * components are submitted by distinct PURL coordinates. As such, there is no 1:1 correlation between total number + * components are submitted by distinct PURLs. As such, there is no 1:1 correlation between total number * of components in the portfolio or project, and records submitted for analysis. */ public class RepositoryMetaAnalyzerTask implements Subscriber { @@ -136,13 +137,13 @@ private void processPortfolio() throws Exception { private void dispatchComponents(final List components) { for (final var component : components) { - kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(null, component.purlCoordinates(), component.internal(), FetchMeta.FETCH_META_LATEST_VERSION)); + kafkaEventDispatcher.dispatchEvent(new ComponentRepositoryMetaAnalysisEvent(component.purl(), component.internal())); } } private List fetchNextComponentsPage(final PersistenceManager pm, final Project project, final long offset) throws Exception { try (final Query query = pm.newQuery(Component.class)) { - var filter = "project.active == :projectActive && purlCoordinates != null"; + var filter = "project.active == :projectActive && purl != null"; var params = new HashMap(); params.put("projectActive", true); if (project != null) { @@ -151,14 +152,19 @@ private List fetchNextComponentsPage(final PersistenceManag } query.setFilter(filter); query.setNamedParameters(params); - query.setOrdering("purlCoordinates ASC"); // Keep the order somewhat consistent + query.setOrdering("purl ASC"); // Keep the order somewhat consistent query.setRange(offset, offset + 5000); - query.setResult("DISTINCT purlCoordinates, internal"); + query.setResult("DISTINCT purl, internal"); return List.copyOf(query.executeResultList(ComponentProjection.class)); } } - public record ComponentProjection(String purlCoordinates, Boolean internal) { + public record ComponentProjection(PackageURL purl, Boolean internal) { + + public ComponentProjection(String purl, Boolean internal) { + this(PurlUtil.silentPurl(purl), internal); + } + } } diff --git a/src/main/java/org/dependencytrack/tasks/TaskScheduler.java b/src/main/java/org/dependencytrack/tasks/TaskScheduler.java index 8a86d789c..f288522ab 100644 --- a/src/main/java/org/dependencytrack/tasks/TaskScheduler.java +++ b/src/main/java/org/dependencytrack/tasks/TaskScheduler.java @@ -30,7 +30,6 @@ import org.dependencytrack.event.EpssMirrorEvent; import org.dependencytrack.event.FortifySscUploadEventAbstract; import org.dependencytrack.event.GitHubAdvisoryMirrorEvent; -import org.dependencytrack.event.IntegrityMetaInitializerEvent; import org.dependencytrack.event.InternalComponentIdentificationEvent; import org.dependencytrack.event.KennaSecurityUploadEventAbstract; import org.dependencytrack.event.NistMirrorEvent; @@ -53,7 +52,6 @@ import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_COMPONENT_IDENTIFICATION_TASK; import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_EPSS_MIRRORING_TASK; import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_GITHUB_MIRRORING_TASK; -import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_INTEGRITY_META_INITIALIZER_TASK; import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_LDAP_SYNC_TASK; import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_NIST_MIRRORING_TASK; import static org.dependencytrack.common.ConfigKey.CRON_EXPRESSION_FOR_OSV_MIRRORING_TASK; @@ -97,8 +95,7 @@ private TaskScheduler() { Map.entry(new PortfolioVulnerabilityAnalysisEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_VULN_ANALYSIS_TASK))), Map.entry(new VulnerabilityScanCleanupEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_VULN_SCAN_CLEANUP_TASK))), Map.entry(new PortfolioRepositoryMetaAnalysisEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_REPO_META_ANALYSIS_TASK))), - Map.entry(new WorkflowStateCleanupEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_WORKFLOW_STATE_CLEANUP_TASK))), - Map.entry(new IntegrityMetaInitializerEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_INTEGRITY_META_INITIALIZER_TASK))) + Map.entry(new WorkflowStateCleanupEvent(), Schedule.create(configInstance.getProperty(CRON_EXPRESSION_FOR_WORKFLOW_STATE_CLEANUP_TASK))) ); if (isTaskEnabled(FORTIFY_SSC_ENABLED)) { diff --git a/src/main/java/org/dependencytrack/util/LockProvider.java b/src/main/java/org/dependencytrack/util/LockProvider.java index 9b401ec05..12bf4470e 100644 --- a/src/main/java/org/dependencytrack/util/LockProvider.java +++ b/src/main/java/org/dependencytrack/util/LockProvider.java @@ -36,8 +36,6 @@ import java.time.Duration; import java.time.Instant; -import static org.dependencytrack.common.ConfigKey.INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR; -import static org.dependencytrack.common.ConfigKey.INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_COMPONENT_IDENTIFICATION_LOCK_AT_LEAST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_COMPONENT_IDENTIFICATION_LOCK_AT_MOST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR; @@ -57,7 +55,6 @@ import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_LEAST_FOR; import static org.dependencytrack.common.ConfigKey.TASK_WORKFLOW_STEP_CLEANUP_LOCK_AT_MOST_FOR; import static org.dependencytrack.tasks.LockName.EPSS_MIRROR_TASK_LOCK; -import static org.dependencytrack.tasks.LockName.INTEGRITY_META_INITIALIZER_LOCK; import static org.dependencytrack.tasks.LockName.INTERNAL_COMPONENT_IDENTIFICATION_TASK_LOCK; import static org.dependencytrack.tasks.LockName.LDAP_SYNC_TASK_LOCK; import static org.dependencytrack.tasks.LockName.PORTFOLIO_METRICS_TASK_LOCK; @@ -170,10 +167,6 @@ public static LockConfiguration getLockConfigurationByLockName(LockName lockName PORTFOLIO_VULN_ANALYSIS_TASK_LOCK.name(), Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_MOST_FOR)), Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_PORTFOLIO_VULN_ANALYSIS_LOCK_AT_LEAST_FOR))); - case INTEGRITY_META_INITIALIZER_LOCK -> new LockConfiguration(Instant.now(), - INTEGRITY_META_INITIALIZER_LOCK.name(), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(INTEGRITY_META_INITIALIZER_LOCK_AT_MOST_FOR)), - Duration.ofMillis(Config.getInstance().getPropertyAsInt(INTEGRITY_META_INITIALIZER_LOCK_AT_LEAST_FOR))); case VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK -> new LockConfiguration(Instant.now(), VULNERABILITY_POLICY_BUNDLE_FETCH_TASK_LOCK.name(), Duration.ofMillis(Config.getInstance().getPropertyAsInt(TASK_VULNERABILITY_POLICY_BUNDLE_FETCH_LOCK_AT_MOST_FOR)), diff --git a/src/main/proto/org/dependencytrack/repometaanalysis/v1/repo_meta_analysis.proto b/src/main/proto/org/dependencytrack/repometaanalysis/v1/repo_meta_analysis.proto index 111dab876..3468435c7 100644 --- a/src/main/proto/org/dependencytrack/repometaanalysis/v1/repo_meta_analysis.proto +++ b/src/main/proto/org/dependencytrack/repometaanalysis/v1/repo_meta_analysis.proto @@ -11,15 +11,8 @@ option java_package = "org.dependencytrack.proto.repometaanalysis.v1"; message AnalysisCommand { // The component that shall be analyzed. Component component = 1; - FetchMeta fetch_meta = 2; -} - -enum FetchMeta{ - FETCH_META_UNSPECIFIED = 0; - FETCH_META_INTEGRITY_DATA = 1; - FETCH_META_LATEST_VERSION = 2; - FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION = 3; + reserved 2; // fetch_meta; removed in 5.6.0 } message AnalysisResult { @@ -34,17 +27,34 @@ message AnalysisResult { // When the latest version was published. optional google.protobuf.Timestamp published = 4; + optional IntegrityMeta integrity_meta = 5; + + // When the latest version information was fetched from the repository. + optional google.protobuf.Timestamp fetched_at = 6; } message IntegrityMeta { + // The MD5 hash of the component. optional string md5 = 1; + + // The SHA1 hash of the component. optional string sha1 = 2; + + // The SHA256 hash of the component. optional string sha256 = 3; + + // The SHA512 hash of the component. optional string sha512 = 4; - // When the component current version last modified. + + // When the component was published. optional google.protobuf.Timestamp current_version_last_modified = 5; + + // URL from where the information was sourced. string meta_source_url = 6; + + // When the integrity metadata was fetched. + google.protobuf.Timestamp fetched_at = 7; } message Component { diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 57d90f4d8..501127598 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -799,13 +799,18 @@ alpine.kafka.processor.epss.mirror.max.batch.size=500 # @category: Kafka # @type: integer # @required -alpine.kafka.processor.repo.meta.analysis.result.max.concurrency=-1 +alpine.kafka.processor.repo.meta.analysis.result.max.batch.size=100 + +# @category: Kafka +# @type: integer +# @required +alpine.kafka.processor.repo.meta.analysis.result.max.concurrency=1 # @category: Kafka # @type: enum # @valid-values: [key, partition, unordered] # @required -alpine.kafka.processor.repo.meta.analysis.result.processing.order=key +alpine.kafka.processor.repo.meta.analysis.result.processing.order=unordered # @category: Kafka # @type: integer @@ -838,6 +843,16 @@ alpine.kafka.processor.repo.meta.analysis.result.consumer.group.id=dtrack-apiser # @required alpine.kafka.processor.repo.meta.analysis.result.consumer.auto.offset.reset=earliest +# @category: Kafka +# @type: integer +# @required +alpine.kafka.processor.repo.meta.analysis.result.consumer.fetch.min.bytes=524288 + +# @category: Kafka +# @type: integer +# @required +alpine.kafka.processor.repo.meta.analysis.result.consumer.fetch.max.wait.ms=1000 + # @category: Kafka # @type: integer # @required diff --git a/src/main/resources/migration/changelog-main.xml b/src/main/resources/migration/changelog-main.xml index 5e7614397..e55e42262 100644 --- a/src/main/resources/migration/changelog-main.xml +++ b/src/main/resources/migration/changelog-main.xml @@ -12,5 +12,6 @@ + \ No newline at end of file diff --git a/src/main/resources/migration/changelog-procedures.xml b/src/main/resources/migration/changelog-procedures.xml index 9eec6ff19..d83a53983 100644 --- a/src/main/resources/migration/changelog-procedures.xml +++ b/src/main/resources/migration/changelog-procedures.xml @@ -20,4 +20,10 @@ + + + + + + \ No newline at end of file diff --git a/src/main/resources/migration/changelog-v5.6.0.xml b/src/main/resources/migration/changelog-v5.6.0.xml new file mode 100644 index 000000000..7275ddee0 --- /dev/null +++ b/src/main/resources/migration/changelog-v5.6.0.xml @@ -0,0 +1,32 @@ + + + + + + + CREATE UNIQUE INDEX "REPOSITORY_META_COMPONENT_REPOSITORY_TYPE_NAME_IDX" + ON "REPOSITORY_META_COMPONENT" ("REPOSITORY_TYPE", "NAME") + WHERE "NAMESPACE" IS NULL; + CREATE UNIQUE INDEX "REPOSITORY_META_COMPONENT_REPOSITORY_TYPE_NAMESPACE_NAME_IDX" + ON "REPOSITORY_META_COMPONENT" ("REPOSITORY_TYPE", "NAMESPACE", "NAME") + WHERE "NAMESPACE" IS NOT NULL; + + + + + + + \ No newline at end of file diff --git a/src/main/resources/migration/procedures/function_component-integrity-check-status.sql b/src/main/resources/migration/procedures/function_component-integrity-check-status.sql new file mode 100644 index 000000000..329bb707a --- /dev/null +++ b/src/main/resources/migration/procedures/function_component-integrity-check-status.sql @@ -0,0 +1,19 @@ +CREATE OR REPLACE FUNCTION "COMPONENT_INTEGRITY_CHECK_STATUS"( + "md5_status" TEXT +, "sha1_status" TEXT +, "sha256_status" TEXT +, "sha512_status" TEXT +) RETURNS TEXT + LANGUAGE "sql" + PARALLEL SAFE + IMMUTABLE +AS +$$ + SELECT CASE + WHEN 'COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN' = ALL(ARRAY["md5_status", "sha1_status", "sha256_status", "sha512_status"]) THEN 'COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN' + WHEN ARRAY['COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN', 'COMPONENT_MISSING_HASH'] <@ ARRAY["md5_status", "sha1_status", "sha256_status", "sha512_status"] THEN 'COMPONENT_MISSING_HASH' + WHEN ARRAY['COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN', 'HASH_MATCH_UNKNOWN'] <@ ARRAY["md5_status", "sha1_status", "sha256_status", "sha512_status"] THEN 'HASH_MATCH_UNKNOWN' + WHEN 'HASH_MATCH_FAILED' = ANY(ARRAY["md5_status", "sha1_status", "sha256_status", "sha512_status"]) THEN 'HASH_MATCH_FAILED' + ELSE 'HASH_MATCH_PASSED' + END +$$; \ No newline at end of file diff --git a/src/main/resources/migration/procedures/function_match-component-hash.sql b/src/main/resources/migration/procedures/function_match-component-hash.sql new file mode 100644 index 000000000..50ee59fbf --- /dev/null +++ b/src/main/resources/migration/procedures/function_match-component-hash.sql @@ -0,0 +1,17 @@ +CREATE OR REPLACE FUNCTION "MATCH_COMPONENT_HASH"( + "actual" TEXT +, "expected" TEXT +) RETURNS TEXT + LANGUAGE "sql" + PARALLEL SAFE + IMMUTABLE +AS +$$ + SELECT CASE + WHEN "actual" IS NULL AND "expected" IS NULL THEN 'COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN' + WHEN "actual" IS NULL THEN 'COMPONENT_MISSING_HASH' + WHEN "expected" IS NULL THEN 'HASH_MATCH_UNKNOWN' + WHEN "actual" = "expected" THEN 'HASH_MATCH_PASSED' + ELSE 'HASH_MATCH_FAILED' + END +$$; \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 41fa2eaac..695fc04d5 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -52,9 +52,6 @@ org.dependencytrack.event.kafka.processor.ProcessorInitializer - - org.dependencytrack.event.PurlMigrator - WhitelistUrlFilter diff --git a/src/test/java/org/dependencytrack/event/PurlMigratorTest.java b/src/test/java/org/dependencytrack/event/PurlMigratorTest.java deleted file mode 100644 index 9166e4380..000000000 --- a/src/test/java/org/dependencytrack/event/PurlMigratorTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event; - -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.junit.Before; -import org.junit.Test; - -import java.sql.Date; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.dependencytrack.model.FetchStatus.IN_PROGRESS; -import static org.dependencytrack.model.FetchStatus.PROCESSED; - -public class PurlMigratorTest extends PersistenceCapableTest { - - final Component componentPersisted = new Component(); - - @Before - public void persistComponentData() { - final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); - componentPersisted.setProject(projectA); - componentPersisted.setName("acme-lib-a"); - componentPersisted.setInternal(false); - componentPersisted.setPurlCoordinates("pkg:maven/acme/acme-lib-a@1.0.1"); - componentPersisted.setPurl("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); - qm.persist(componentPersisted); - kafkaMockProducer.clear(); - } - - @Test - public void testIntegrityMetaInitializerWhenDisabledByDefault() { - PurlMigrator initializer = new PurlMigrator(false); - initializer.contextInitialized(null); - assertThat(qm.getIntegrityMetaComponentCount()).isZero(); - assertThat(kafkaMockProducer.history().size()).isZero(); - } - - @Test - public void testIntegrityMetaInitializerWithExistingDataProcessed() { - var integrityMetaExisting = new IntegrityMetaComponent(); - integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); - integrityMetaExisting.setStatus(PROCESSED); - qm.persist(integrityMetaExisting); - // data exists in IntegrityMetaComponent so sync will be skipped - PurlMigrator initializer = new PurlMigrator(true); - initializer.contextInitialized(null); - // kafka event is not dispatched - assertThat(kafkaMockProducer.history().size()).isZero(); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - } - - - @Test - public void testIntegrityMetaInitializerWithExistingDataFetchedRecently() { - // data exists in IntegrityMetaComponent but last fetched 30 min ago < 1 hour wait time - var integrityMetaExisting = new IntegrityMetaComponent(); - integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); - integrityMetaExisting.setStatus(IN_PROGRESS); - integrityMetaExisting.setLastFetch(Date.from(Instant.now().minus(30, ChronoUnit.MINUTES))); - qm.persist(integrityMetaExisting); - - PurlMigrator initializer = new PurlMigrator(true); - initializer.contextInitialized(null); - // kafka event is dispatched - assertThat(kafkaMockProducer.history().size()).isZero(); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - } -} diff --git a/src/test/java/org/dependencytrack/event/kafka/KafkaEventDispatcherTest.java b/src/test/java/org/dependencytrack/event/kafka/KafkaEventDispatcherTest.java index 0c2b3bf6b..f65dda6e1 100644 --- a/src/test/java/org/dependencytrack/event/kafka/KafkaEventDispatcherTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/KafkaEventDispatcherTest.java @@ -20,6 +20,7 @@ import alpine.notification.Notification; import alpine.notification.NotificationLevel; +import com.github.packageurl.PackageURL; import org.apache.kafka.clients.producer.MockProducer; import org.apache.kafka.clients.producer.RecordMetadata; import org.apache.kafka.common.serialization.ByteArraySerializer; @@ -33,7 +34,6 @@ import org.dependencytrack.model.VulnerabilityAnalysisLevel; import org.dependencytrack.notification.NotificationGroup; import org.dependencytrack.notification.NotificationScope; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; import org.junit.Before; import org.junit.Test; @@ -64,16 +64,15 @@ public void testDispatchEventWithNull() { } @Test - public void testDispatchEventWithComponentRepositoryMetaAnalysisEvent() { - final var event = new ComponentRepositoryMetaAnalysisEvent(UUID.randomUUID(), - "pkg:maven/foo/bar@1.2.3", /* internal */ false, FetchMeta.FETCH_META_LATEST_VERSION); + public void testDispatchEventWithComponentRepositoryMetaAnalysisEvent() throws Exception { + final var event = new ComponentRepositoryMetaAnalysisEvent(new PackageURL("pkg:maven/foo/bar@1.2.3"), /* internal */ false); final CompletableFuture future = eventDispatcher.dispatchEvent(event); assertThat(mockProducer.completeNext()).isTrue(); assertThat(future).isCompletedWithValueMatching(Objects::nonNull); assertThat(mockProducer.history()).satisfiesExactly(record -> { assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); - assertThat(record.key()).asString().isEqualTo("pkg:maven/foo/bar@1.2.3"); + assertThat(record.key()).asString().isEqualTo("pkg:maven/foo/bar"); assertThat(record.value()).isNotNull(); assertThat(record.headers()).isEmpty(); }); diff --git a/src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java b/src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java deleted file mode 100644 index 008090bcb..000000000 --- a/src/test/java/org/dependencytrack/event/kafka/componentmeta/HandlerFactoryTest.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import alpine.common.logging.Logger; -import com.github.packageurl.MalformedPackageURLException; -import com.github.packageurl.PackageURL; -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; -import org.dependencytrack.util.PurlUtil; -import org.junit.Test; - -import java.util.UUID; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class HandlerFactoryTest extends PersistenceCapableTest { - - private static final Logger LOGGER = Logger.getLogger(HandlerFactoryTest.class); - - @Test - public void createHandlerForSupportedPackageTest() { - Handler handler; - KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); - try { - PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); - ComponentProjection componentProjection = new ComponentProjection(UUID.randomUUID(), PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl); - handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION); - assertTrue(handler instanceof SupportedMetaHandler); - } catch (MalformedPackageURLException e) { - LOGGER.warn("Package url not formed correctly"); - } - - } - - @Test - public void createHandlerForUnSupportedPackageTest() { - Handler handler; - KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); - try { - PackageURL packageUrl = new PackageURL("pkg:golang/github.com/foo/bar@1.2.3"); - ComponentProjection componentProjection = new ComponentProjection(UUID.randomUUID(), PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl); - handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_LATEST_VERSION); - assertTrue(handler instanceof UnSupportedMetaHandler); - } catch (MalformedPackageURLException e) { - throw new RuntimeException(e); - } - - } -} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java b/src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java deleted file mode 100644 index c888bfab5..000000000 --- a/src/test/java/org/dependencytrack/event/kafka/componentmeta/SupportedMetaHandlerTest.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import com.github.packageurl.MalformedPackageURLException; -import com.github.packageurl.PackageURL; -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.event.kafka.KafkaTopics; -import org.dependencytrack.model.FetchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; -import org.dependencytrack.util.PurlUtil; -import org.junit.Test; -import org.junit.jupiter.api.Assertions; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Date; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.dependencytrack.util.KafkaTestUtil.deserializeValue; - -public class SupportedMetaHandlerTest extends PersistenceCapableTest { - - @Test - public void testHandleIntegrityComponentNotInDB() throws MalformedPackageURLException { - Handler handler; - UUID uuid = UUID.randomUUID(); - KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); - PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); - ComponentProjection componentProjection = new ComponentProjection(uuid, PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl); - IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl().toString()); - Assertions.assertNull(integrityMetaComponent); - handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA); - IntegrityMetaComponent result = handler.handle(); - assertThat(kafkaMockProducer.history()).satisfiesExactly( - record -> { - assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); - final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); - assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/org.http4s/blaze-core_2.12"); - assertThat(command.getComponent().getUuid()).isEqualTo(uuid.toString()); - assertThat(command.getComponent().getInternal()).isFalse(); - assertThat(command.getFetchMeta()).isEqualTo(FetchMeta.FETCH_META_INTEGRITY_DATA); - } - - ); - Assertions.assertEquals(FetchStatus.IN_PROGRESS, result.getStatus()); - } - - @Test - public void testHandleIntegrityComponentInDBForMoreThanAnHour() throws MalformedPackageURLException { - Handler handler; - UUID uuid = UUID.randomUUID(); - KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); - PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); - ComponentProjection componentProjection = new ComponentProjection(uuid, PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl); - var integrityMeta = new IntegrityMetaComponent(); - integrityMeta.setPurl("pkg:maven/org.http4s/blaze-core_2.12"); - integrityMeta.setStatus(FetchStatus.IN_PROGRESS); - integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); - qm.createIntegrityMetaComponent(integrityMeta); - handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA); - IntegrityMetaComponent integrityMetaComponent = handler.handle(); - assertThat(kafkaMockProducer.history()).satisfiesExactly( - record -> { - assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); - final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); - assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/org.http4s/blaze-core_2.12"); - assertThat(command.getComponent().getUuid()).isEqualTo(uuid.toString()); - assertThat(command.getComponent().getInternal()).isFalse(); - assertThat(command.getFetchMeta()).isEqualTo(FetchMeta.FETCH_META_INTEGRITY_DATA); - } - - ); - Assertions.assertEquals(FetchStatus.IN_PROGRESS, integrityMetaComponent.getStatus()); - assertThat(integrityMetaComponent.getLastFetch()).isAfter(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); - } - - @Test - public void testHandleIntegrityWhenMetadataExists() throws MalformedPackageURLException { - Handler handler; - UUID uuid = UUID.randomUUID(); - KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); - PackageURL packageUrl = new PackageURL("pkg:maven/org.http4s/blaze-core_2.12"); - ComponentProjection componentProjection = new ComponentProjection(uuid, PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl); - var integrityMeta = new IntegrityMetaComponent(); - integrityMeta.setPurl("pkg:maven/org.http4s/blaze-core_2.12"); - integrityMeta.setMd5("md5hash"); - integrityMeta.setStatus(FetchStatus.PROCESSED); - integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); - qm.createIntegrityMetaComponent(integrityMeta); - handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION); - IntegrityMetaComponent integrityMetaComponent = handler.handle(); - assertThat(kafkaMockProducer.history()).satisfiesExactly( - record -> { - assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); - final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); - assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/org.http4s/blaze-core_2.12"); - assertThat(command.getComponent().getUuid()).isEqualTo(uuid.toString()); - assertThat(command.getComponent().getInternal()).isFalse(); - assertThat(command.getFetchMeta()).isEqualTo(FetchMeta.FETCH_META_LATEST_VERSION); - } - - ); - Assertions.assertEquals(FetchStatus.PROCESSED, integrityMetaComponent.getStatus()); - } -} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java b/src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java deleted file mode 100644 index 8d54704e3..000000000 --- a/src/test/java/org/dependencytrack/event/kafka/componentmeta/UnSupportedMetaHandlerTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.event.kafka.componentmeta; - -import com.github.packageurl.MalformedPackageURLException; -import com.github.packageurl.PackageURL; -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.kafka.KafkaEventDispatcher; -import org.dependencytrack.event.kafka.KafkaTopics; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.proto.repometaanalysis.v1.FetchMeta; -import org.dependencytrack.util.PurlUtil; -import org.junit.Test; -import org.junit.jupiter.api.Assertions; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.dependencytrack.util.KafkaTestUtil.deserializeValue; - -public class UnSupportedMetaHandlerTest extends PersistenceCapableTest { - - @Test - public void testHandleComponentInDb() throws MalformedPackageURLException { - Handler handler; - KafkaEventDispatcher kafkaEventDispatcher = new KafkaEventDispatcher(); - PackageURL packageUrl = new PackageURL("pkg:golang/foo/bar@baz?ping=pong#1/2/3"); - ComponentProjection componentProjection = new ComponentProjection(null, PurlUtil.silentPurlCoordinatesOnly(packageUrl).toString(), false, packageUrl); - IntegrityMetaComponent integrityMetaComponent = qm.getIntegrityMetaComponent(componentProjection.purl().toString()); - Assertions.assertNull(integrityMetaComponent); - handler = HandlerFactory.createHandler(componentProjection, qm, kafkaEventDispatcher, FetchMeta.FETCH_META_LATEST_VERSION); - handler.handle(); - assertThat(kafkaMockProducer.history()).satisfiesExactly( - record -> { - assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); - final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); - assertThat(command.getComponent().getPurl()).isEqualTo("pkg:golang/foo/bar@baz"); - assertThat(command.getComponent().getInternal()).isFalse(); - assertThat(command.getFetchMeta()).isEqualTo(FetchMeta.FETCH_META_LATEST_VERSION); - } - - ); - Assertions.assertNull(integrityMetaComponent); - } -} \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java b/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java index db01caaf5..ecc59d7a4 100644 --- a/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java +++ b/src/test/java/org/dependencytrack/event/kafka/processor/RepositoryMetaResultProcessorTest.java @@ -38,6 +38,7 @@ import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Date; +import java.util.List; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -68,7 +69,7 @@ public void processNewMetaModelTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar", result).build())); final RepositoryMetaComponent metaComponent = qm.getRepositoryMetaComponent(RepositoryType.MAVEN, "foo", "bar"); @@ -89,7 +90,7 @@ public void processWithoutComponentDetailsTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar", result).build())); final Query query = qm.getPersistenceManager().newQuery(RepositoryMetaComponent.class); query.setResult("count(this)"); @@ -119,7 +120,7 @@ public void processUpdateExistingMetaModelTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar", result).build())); qm.getPersistenceManager().refresh(metaComponent); assertThat(metaComponent).isNotNull(); @@ -155,7 +156,7 @@ public void processUpdateOutOfOrderMetaModelTest() throws Exception { // Pipe in a record that was produced 10 seconds ago, 5 seconds before metaComponent's lastCheck. final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).withTimestamp(Instant.now().minusSeconds(10)).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).withTimestamp(Instant.now().minusSeconds(10)).build())); qm.getPersistenceManager().refresh(metaComponent); assertThat(metaComponent).isNotNull(); @@ -201,9 +202,9 @@ public void processUpdateIntegrityResultTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); qm.getPersistenceManager().refresh(integrityMetaComponent); - integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent = getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); assertThat(integrityMetaComponent.getMd5()).isEqualTo("098f6bcd4621d373cade4e832627b4f6"); assertThat(integrityMetaComponent.getSha1()).isEqualTo("a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"); @@ -211,7 +212,7 @@ public void processUpdateIntegrityResultTest() throws Exception { assertThat(integrityMetaComponent.getLastFetch()).isAfter(date); assertThat(integrityMetaComponent.getStatus()).isEqualTo(FetchStatus.PROCESSED); - IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); + IntegrityAnalysis analysis = getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis.getIntegrityCheckStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_PASSED); assertThat(analysis.getMd5HashMatchStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_PASSED); assertThat(analysis.getSha1HashMatchStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_PASSED); @@ -254,15 +255,15 @@ public void testIntegrityCheckWhenComponentHashIsMissing() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); qm.getPersistenceManager().refresh(integrityMetaComponent); - integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent = getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); assertThat(integrityMetaComponent.getRepositoryUrl()).isEqualTo("test"); assertThat(integrityMetaComponent.getLastFetch()).isAfter(date); assertThat(integrityMetaComponent.getStatus()).isEqualTo(FetchStatus.NOT_AVAILABLE); - IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); + IntegrityAnalysis analysis = getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis.getIntegrityCheckStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_UNKNOWN); assertThat(analysis.getMd5HashMatchStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_UNKNOWN); assertThat(analysis.getSha1HashMatchStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_UNKNOWN); @@ -305,9 +306,9 @@ public void testIntegrityAnalysisWillNotBePerformedIfNoIntegrityDataInResult() t .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); - IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); + IntegrityAnalysis analysis = getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis).isNull(); } @@ -344,9 +345,9 @@ public void testIntegrityCheckWillNotBeDoneIfComponentUuidAndIntegrityDataIsMiss .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); - IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); + IntegrityAnalysis analysis = getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis).isNull(); } @@ -384,9 +385,9 @@ public void testIntegrityIfResultHasIntegrityDataAndComponentUuidIsMissing() thr .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); - IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); + IntegrityAnalysis analysis = getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis).isNotNull(); assertThat(analysis.getComponent()).isEqualTo(c); assertThat(analysis.getIntegrityCheckStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_PASSED); @@ -415,9 +416,9 @@ public void testIntegrityCheckWillNotBeDoneIfComponentIsNotInDb() throws Excepti .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); - IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(uuid); + IntegrityAnalysis analysis = getIntegrityAnalysisByComponentUuid(uuid); assertThat(analysis).isNull(); } @@ -437,7 +438,7 @@ public void testIntegrityCheckShouldReturnComponentHashMissing() throws Exceptio Component c = qm.persist(componentProjectA); var integrityMetaComponent = new IntegrityMetaComponent(); - integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3?foo=bar"); integrityMetaComponent.setStatus(FetchStatus.IN_PROGRESS); Date date = Date.from(Instant.now().minus(15, ChronoUnit.MINUTES)); integrityMetaComponent.setLastFetch(date); @@ -446,16 +447,16 @@ public void testIntegrityCheckShouldReturnComponentHashMissing() throws Exceptio final var result = AnalysisResult.newBuilder() .setComponent(org.dependencytrack.proto.repometaanalysis.v1.Component.newBuilder() .setUuid(c.getUuid().toString()) - .setPurl("pkg:maven/foo/bar@1.2.3")) + .setPurl("pkg:maven/foo/bar@1.2.3?foo=bar")) .setIntegrityMeta(IntegrityMeta.newBuilder().setMd5("098f6bcd4621d373cade4e832627b4f6") .setSha1("a94a8fe5ccb19ba61c4c0873d391e987982fbbd3") .setMetaSourceUrl("test").build()) .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); qm.getPersistenceManager().refresh(integrityMetaComponent); - integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent = getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3?foo=bar"); assertThat(integrityMetaComponent).isNotNull(); assertThat(integrityMetaComponent.getMd5()).isEqualTo("098f6bcd4621d373cade4e832627b4f6"); assertThat(integrityMetaComponent.getSha1()).isEqualTo("a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"); @@ -463,7 +464,7 @@ public void testIntegrityCheckShouldReturnComponentHashMissing() throws Exceptio assertThat(integrityMetaComponent.getLastFetch()).isAfter(date); assertThat(integrityMetaComponent.getStatus()).isEqualTo(FetchStatus.PROCESSED); - IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); + IntegrityAnalysis analysis = getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis.getIntegrityCheckStatus()).isEqualTo(IntegrityMatchStatus.COMPONENT_MISSING_HASH); assertThat(analysis.getMd5HashMatchStatus()).isEqualTo(IntegrityMatchStatus.COMPONENT_MISSING_HASH); assertThat(analysis.getSha1HashMatchStatus()).isEqualTo(IntegrityMatchStatus.COMPONENT_MISSING_HASH); @@ -487,7 +488,7 @@ public void testIntegrityCheckShouldReturnComponentHashMissingAndMatchUnknown() Component c = qm.persist(componentProjectA); var integrityMetaComponent = new IntegrityMetaComponent(); - integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3?foo=bar"); integrityMetaComponent.setStatus(FetchStatus.IN_PROGRESS); Date date = Date.from(Instant.now().minus(15, ChronoUnit.MINUTES)); integrityMetaComponent.setLastFetch(date); @@ -496,21 +497,21 @@ public void testIntegrityCheckShouldReturnComponentHashMissingAndMatchUnknown() final var result = AnalysisResult.newBuilder() .setComponent(org.dependencytrack.proto.repometaanalysis.v1.Component.newBuilder() .setUuid(c.getUuid().toString()) - .setPurl("pkg:maven/foo/bar@1.2.3")) + .setPurl("pkg:maven/foo/bar@1.2.3?foo=bar")) .setIntegrityMeta(IntegrityMeta.newBuilder() .setMetaSourceUrl("test").build()) .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); qm.getPersistenceManager().refresh(integrityMetaComponent); - integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent = getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3?foo=bar"); assertThat(integrityMetaComponent).isNotNull(); assertThat(integrityMetaComponent.getRepositoryUrl()).isEqualTo("test"); assertThat(integrityMetaComponent.getLastFetch()).isAfter(date); assertThat(integrityMetaComponent.getStatus()).isEqualTo(FetchStatus.NOT_AVAILABLE); - IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); + IntegrityAnalysis analysis = getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis.getIntegrityCheckStatus()).isEqualTo(IntegrityMatchStatus.COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN); assertThat(analysis.getMd5HashMatchStatus()).isEqualTo(IntegrityMatchStatus.COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN); assertThat(analysis.getSha1HashMatchStatus()).isEqualTo(IntegrityMatchStatus.COMPONENT_MISSING_HASH_AND_MATCH_UNKNOWN); @@ -536,7 +537,7 @@ public void testIntegrityCheckShouldFailIfNoHashMatch() throws Exception { Component c = qm.persist(componentProjectA); var integrityMetaComponent = new IntegrityMetaComponent(); - integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3?foo=bar"); integrityMetaComponent.setStatus(FetchStatus.IN_PROGRESS); Date date = Date.from(Instant.now().minus(15, ChronoUnit.MINUTES)); integrityMetaComponent.setLastFetch(date); @@ -545,7 +546,7 @@ public void testIntegrityCheckShouldFailIfNoHashMatch() throws Exception { final var result = AnalysisResult.newBuilder() .setComponent(org.dependencytrack.proto.repometaanalysis.v1.Component.newBuilder() .setUuid(c.getUuid().toString()) - .setPurl("pkg:maven/foo/bar@1.2.3")) + .setPurl("pkg:maven/foo/bar@1.2.3?foo=bar")) .setIntegrityMeta(IntegrityMeta.newBuilder() .setSha1("somevalue") .setMd5("someothervalue") @@ -553,9 +554,9 @@ public void testIntegrityCheckShouldFailIfNoHashMatch() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); qm.getPersistenceManager().refresh(integrityMetaComponent); - integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent = getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3?foo=bar"); assertThat(integrityMetaComponent).isNotNull(); assertThat(integrityMetaComponent.getRepositoryUrl()).isEqualTo("test"); assertThat(integrityMetaComponent.getMd5()).isEqualTo("someothervalue"); @@ -563,7 +564,7 @@ public void testIntegrityCheckShouldFailIfNoHashMatch() throws Exception { assertThat(integrityMetaComponent.getLastFetch()).isAfter(date); assertThat(integrityMetaComponent.getStatus()).isEqualTo(FetchStatus.PROCESSED); - IntegrityAnalysis analysis = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); + IntegrityAnalysis analysis = getIntegrityAnalysisByComponentUuid(c.getUuid()); assertThat(analysis.getIntegrityCheckStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_FAILED); assertThat(analysis.getMd5HashMatchStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_FAILED); assertThat(analysis.getSha1HashMatchStatus()).isEqualTo(IntegrityMatchStatus.HASH_MATCH_FAILED); @@ -588,9 +589,9 @@ public void processUpdateIntegrityResultNotAvailableTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); qm.getPersistenceManager().refresh(integrityMetaComponent); - integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent = getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); assertThat(integrityMetaComponent.getMd5()).isNull(); assertThat(integrityMetaComponent.getSha1()).isNull(); @@ -615,14 +616,16 @@ public void processUpdateOldIntegrityResultSent() throws Exception { final var result = AnalysisResult.newBuilder() .setComponent(org.dependencytrack.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:maven/foo/bar@1.2.3")) - .setIntegrityMeta(IntegrityMeta.newBuilder().setMd5("098f6bcd4621d373cade4e832627b4f6") - .setSha1("a94a8fe5ccb19ba61c4c0873d391e587982fbbd3").setMetaSourceUrl("test2").build()) + .setIntegrityMeta(IntegrityMeta.newBuilder() + .setMd5("098f6bcd4621d373cade4e832627b4f6") + .setSha1("a94a8fe5ccb19ba61c4c0873d391e587982fbbd3") + .setMetaSourceUrl("test2")) .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).withTimestamp(Instant.now().minus(30, ChronoUnit.MINUTES)).build())); qm.getPersistenceManager().refresh(integrityMetaComponent); - integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent = getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); assertThat(integrityMetaComponent.getLastFetch()).isEqualTo(date); assertThat(integrityMetaComponent.getMd5()).isEqualTo("098f6bcd4621d373cade4e832627b4f6"); @@ -654,7 +657,7 @@ public void processBothMetaModelAndIntegrityMeta() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar", result).build())); qm.getPersistenceManager().refresh(integrityMetaComponent); final RepositoryMetaComponent metaComponent = qm.getRepositoryMetaComponent(RepositoryType.MAVEN, "foo", "bar"); @@ -689,9 +692,9 @@ public void processUpdateIntegrityResultNotSentTest() throws Exception { .build(); final var processor = new RepositoryMetaResultProcessor(); - processor.process(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build()); + processor.process(List.of(aConsumerRecord("pkg:maven/foo/bar@1.2.3", result).build())); qm.getPersistenceManager().refresh(integrityMetaComponent); - integrityMetaComponent = qm.getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); + integrityMetaComponent = getIntegrityMetaComponent("pkg:maven/foo/bar@1.2.3"); assertThat(integrityMetaComponent).isNotNull(); assertThat(integrityMetaComponent.getMd5()).isNull(); assertThat(integrityMetaComponent.getSha1()).isNull(); @@ -699,4 +702,27 @@ public void processUpdateIntegrityResultNotSentTest() throws Exception { assertThat(integrityMetaComponent.getLastFetch()).isEqualTo(date); assertThat(integrityMetaComponent.getStatus()).isEqualTo(FetchStatus.IN_PROGRESS); } + + private IntegrityMetaComponent getIntegrityMetaComponent(final String purl) { + final Query query = qm.getPersistenceManager().newQuery(IntegrityMetaComponent.class); + query.setFilter("purl == :purl"); + query.setParameters(purl); + try { + return query.executeUnique(); + } finally { + query.closeAll(); + } + } + + private IntegrityAnalysis getIntegrityAnalysisByComponentUuid(final UUID uuid) { + final Query query = qm.getPersistenceManager().newQuery(IntegrityAnalysis.class); + query.setFilter("component.uuid == :uuid"); + query.setParameters(uuid); + try { + return query.executeUnique(); + } finally { + query.closeAll(); + } + } + } \ No newline at end of file diff --git a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerPostgresTest.java b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerPostgresTest.java deleted file mode 100644 index cbf0fac15..000000000 --- a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerPostgresTest.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.persistence; - -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.FetchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.dependencytrack.model.Project; -import org.junit.Test; - -import java.util.Date; - -import static org.assertj.core.api.Assertions.assertThat; - -public class IntegrityMetaQueryManagerPostgresTest extends PersistenceCapableTest { - - @Test - public void testCreateIntegrityMetadataHandlingConflict() { - var integrityMeta = new IntegrityMetaComponent(); - integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.IN_PROGRESS); - integrityMeta.setLastFetch(new Date()); - qm.createIntegrityMetaHandlingConflict(integrityMeta); - - var integrityMeta2 = new IntegrityMetaComponent(); - //inserting same purl twice should not cause exception - integrityMeta2.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta2.setStatus(FetchStatus.IN_PROGRESS); - integrityMeta2.setLastFetch(new Date()); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - } - - @Test - public void testSynchronizeIntegrityMetaComponent() { - final var project = new Project(); - project.setName("acme-app"); - project.setVersion("1.0.0"); - qm.persist(project); - - final var component = new Component(); - component.setProject(project); - component.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - component.setName("acme-lib"); - - // without any component in database - qm.synchronizeIntegrityMetaComponent(); - assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).isNull(); - - // with existing component in database - qm.persist(component); - qm.synchronizeIntegrityMetaComponent(); - assertThat(qm.getIntegrityMetaComponent(component.getPurl().toString())).satisfies( - meta -> { - assertThat(meta.getStatus()).isNull(); - assertThat(meta.getPurl()).isEqualTo("pkg:maven/acme/example@1.0.0?type=jar"); - assertThat(meta.getId()).isGreaterThan(0L); - assertThat(meta.getMd5()).isNull(); - assertThat(meta.getSha1()).isNull(); - assertThat(meta.getSha256()).isNull(); - assertThat(meta.getLastFetch()).isNull(); - assertThat(meta.getPublishedAt()).isNull(); - } - ); - } -} diff --git a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java b/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java deleted file mode 100644 index 68c2161f0..000000000 --- a/src/test/java/org/dependencytrack/persistence/IntegrityMetaQueryManagerTest.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.persistence; - -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.model.FetchStatus; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.junit.Test; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Date; - -import static org.assertj.core.api.Assertions.assertThat; - -public class IntegrityMetaQueryManagerTest extends PersistenceCapableTest { - - @Test - public void testGetIntegrityMetaComponent() { - var integrityMeta = new IntegrityMetaComponent(); - integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.IN_PROGRESS); - integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); - - var result = qm.getIntegrityMetaComponent("pkg:maven/acme/example@1.0.0?type=jar"); - assertThat(result).isNull(); - - result = qm.persist(integrityMeta); - assertThat(qm.getIntegrityMetaComponent(result.getPurl())).satisfies( - meta -> { - assertThat(meta.getStatus()).isEqualTo(FetchStatus.IN_PROGRESS); - assertThat(meta.getMd5()).isNull(); - assertThat(meta.getSha1()).isNull(); - assertThat(meta.getSha256()).isNull(); - assertThat(meta.getLastFetch()).isBefore(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); - assertThat(meta.getPublishedAt()).isNull(); - } - ); - } - - @Test - public void testUpdateIntegrityMetaComponent() { - var integrityMeta = new IntegrityMetaComponent(); - integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.IN_PROGRESS); - integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.MINUTES))); - - var result = qm.updateIntegrityMetaComponent(integrityMeta); - assertThat(result).isNull(); - - var persisted = qm.persist(integrityMeta); - persisted.setStatus(FetchStatus.PROCESSED); - result = qm.updateIntegrityMetaComponent(persisted); - assertThat(result.getStatus()).isEqualTo(FetchStatus.PROCESSED); - } - - @Test - public void testGetIntegrityMetaComponentCount() { - var integrityMeta = new IntegrityMetaComponent(); - integrityMeta.setPurl("pkg:maven/acme/example@1.0.0?type=jar"); - integrityMeta.setStatus(FetchStatus.IN_PROGRESS); - qm.persist(integrityMeta); - - integrityMeta = new IntegrityMetaComponent(); - integrityMeta.setPurl("pkg:npm/acme/example@2.0.0"); - integrityMeta.setStatus(FetchStatus.PROCESSED); - qm.persist(integrityMeta); - - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(2); - } -} diff --git a/src/test/java/org/dependencytrack/persistence/QueryManagerTest.java b/src/test/java/org/dependencytrack/persistence/QueryManagerTest.java index 46b085f3b..4b1fdf81e 100644 --- a/src/test/java/org/dependencytrack/persistence/QueryManagerTest.java +++ b/src/test/java/org/dependencytrack/persistence/QueryManagerTest.java @@ -66,7 +66,7 @@ public void testGetMetaInformation() { integrityMetaComponent.setLastFetch(published); integrityMetaComponent.setStatus(FetchStatus.PROCESSED); integrityMetaComponent.setRepositoryUrl("repo.url.com"); - qm.createIntegrityMetaComponent(integrityMetaComponent); + qm.persist(integrityMetaComponent); component = qm.createComponent(component, false); ComponentMetaInformation componentMetaInformation = qm.getMetaInformation(component.getUuid()); assertEquals(HASH_MATCH_PASSED, componentMetaInformation.integrityMatchStatus()); @@ -95,7 +95,7 @@ public void testGetMetaInformationWhenPublishedAtIsMissing() { IntegrityMetaComponent integrityMetaComponent = new IntegrityMetaComponent(); integrityMetaComponent.setPurl(component.getPurl().toString()); integrityMetaComponent.setStatus(FetchStatus.PROCESSED); - qm.createIntegrityMetaComponent(integrityMetaComponent); + qm.persist(integrityMetaComponent); component = qm.createComponent(component, false); ComponentMetaInformation componentMetaInformation = qm.getMetaInformation(component.getUuid()); assertEquals(HASH_MATCH_PASSED, componentMetaInformation.integrityMatchStatus()); @@ -113,7 +113,7 @@ public void testGetMetaInformationWhenIntregrityAnalysisIsMissing() { IntegrityMetaComponent integrityMetaComponent = new IntegrityMetaComponent(); integrityMetaComponent.setPurl(component.getPurl().toString()); integrityMetaComponent.setStatus(FetchStatus.PROCESSED); - qm.createIntegrityMetaComponent(integrityMetaComponent); + qm.persist(integrityMetaComponent); component = qm.createComponent(component, false); ComponentMetaInformation componentMetaInformation = qm.getMetaInformation(component.getUuid()); assertNull(componentMetaInformation.publishedDate()); diff --git a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java index 0635b5ff4..52e405881 100644 --- a/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/CelPolicyEngineTest.java @@ -425,7 +425,7 @@ public void testEvaluateProjectWithPolicyOperatorForComponentAgeLessThan() throw integrityMetaComponent.setPublishedAt(publishedDate); integrityMetaComponent.setStatus(FetchStatus.PROCESSED); integrityMetaComponent.setLastFetch(new Date()); - qm.createIntegrityMetaComponent(integrityMetaComponent); + qm.persist(integrityMetaComponent); new CelPolicyEngine().evaluateProject(project.getUuid()); assertThat(qm.getAllPolicyViolations(component)).hasSize(1); assertThat(qm.getAllPolicyViolations(component).get(0).getPolicyCondition().getValue()).isEqualTo(""" @@ -492,7 +492,7 @@ public void testEvaluateProjectWithPolicyOperatorForComponentAgeGreaterThan() th integrityMetaComponent.setPublishedAt(publishedDate); integrityMetaComponent.setStatus(FetchStatus.PROCESSED); integrityMetaComponent.setLastFetch(new Date()); - qm.createIntegrityMetaComponent(integrityMetaComponent); + qm.persist(integrityMetaComponent); new CelPolicyEngine().evaluateProject(project.getUuid()); assertThat(qm.getAllPolicyViolations(component)).hasSize(1); diff --git a/src/test/java/org/dependencytrack/policy/cel/compat/ComponentAgeCelPolicyTest.java b/src/test/java/org/dependencytrack/policy/cel/compat/ComponentAgeCelPolicyTest.java index 75329d041..9f4f66610 100644 --- a/src/test/java/org/dependencytrack/policy/cel/compat/ComponentAgeCelPolicyTest.java +++ b/src/test/java/org/dependencytrack/policy/cel/compat/ComponentAgeCelPolicyTest.java @@ -94,7 +94,7 @@ public void evaluateTest(Instant publishedDate, PolicyCondition.Operator operato metaComponent.setPublishedAt(Date.from(publishedDate)); } metaComponent.setLastFetch(new Date()); - qm.createIntegrityMetaComponent(metaComponent); + qm.persist(metaComponent); new CelPolicyEngine().evaluateProject(project.getUuid()); diff --git a/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java b/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java index 7c1ca113e..d3ede57f2 100644 --- a/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java +++ b/src/test/java/org/dependencytrack/resources/v1/ComponentResourceTest.java @@ -42,12 +42,12 @@ import org.junit.ClassRule; import org.junit.Test; -import javax.jdo.JDOObjectNotFoundException; import jakarta.json.JsonArray; import jakarta.json.JsonObject; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import javax.jdo.JDOObjectNotFoundException; import java.time.Instant; import java.util.ArrayList; import java.util.Date; @@ -57,7 +57,6 @@ import static net.javacrumbs.jsonunit.assertj.JsonAssertions.assertThatJson; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; -import static org.dependencytrack.model.IntegrityMatchStatus.HASH_MATCH_FAILED; import static org.dependencytrack.model.IntegrityMatchStatus.HASH_MATCH_PASSED; import static org.dependencytrack.model.IntegrityMatchStatus.HASH_MATCH_UNKNOWN; @@ -210,7 +209,7 @@ public void getComponentByUuidWithPublishedMetaDataTest() { integrityMetaComponent.setPublishedAt(published); integrityMetaComponent.setLastFetch(published); integrityMetaComponent.setStatus(FetchStatus.PROCESSED); - qm.createIntegrityMetaComponent(integrityMetaComponent); + qm.persist(integrityMetaComponent); RepositoryMetaComponent meta = new RepositoryMetaComponent(); Date lastCheck = new Date(); meta.setLastCheck(lastCheck); @@ -239,131 +238,6 @@ public void getComponentByUuidWithPublishedMetaDataTest() { Assert.assertEquals(published.toString(), Date.from(Instant.ofEpochSecond(json.getJsonObject("componentMetaInformation").getJsonNumber("lastFetched").longValue() / 1000)).toString()); } - - @Test - public void integrityCheckStatusPassTest() { - Project project = qm.createProject("Acme Application", null, null, null, null, null, true, false); - Component component = new Component(); - component.setProject(project); - component.setName("ABC"); - component.setPurl("pkg:maven/org.acme/abc"); - IntegrityAnalysis integrityAnalysis = new IntegrityAnalysis(); - integrityAnalysis.setComponent(component); - integrityAnalysis.setIntegrityCheckStatus(IntegrityMatchStatus.HASH_MATCH_PASSED); - Date published = new Date(); - integrityAnalysis.setUpdatedAt(published); - integrityAnalysis.setId(component.getId()); - integrityAnalysis.setMd5HashMatchStatus(IntegrityMatchStatus.HASH_MATCH_PASSED); - integrityAnalysis.setSha1HashMatchStatus(HASH_MATCH_UNKNOWN); - integrityAnalysis.setSha256HashMatchStatus(HASH_MATCH_UNKNOWN); - integrityAnalysis.setSha512HashMatchStatus(HASH_MATCH_PASSED); - qm.persist(integrityAnalysis); - component = qm.createComponent(component, false); - Response response = jersey.target(V1_COMPONENT + "/" + component.getUuid() + "/integritycheckstatus") - .request().header(X_API_KEY, apiKey).get(Response.class); - Assert.assertEquals(200, response.getStatus(), 0); - Assert.assertNull(response.getHeaderString(TOTAL_COUNT_HEADER)); - JsonObject json = parseJsonObject(response); - Assert.assertNotNull(json); - Assert.assertEquals(HASH_MATCH_PASSED.name(), json.getString("md5HashMatchStatus")); - Assert.assertEquals(HASH_MATCH_PASSED.name(), json.getString("integrityCheckStatus")); - Assert.assertEquals(HASH_MATCH_PASSED.name(), json.getString("sha512HashMatchStatus")); - Assert.assertEquals(published.toString(), Date.from(Instant.ofEpochSecond(json.getJsonNumber("updatedAt").longValue() / 1000)).toString()); - } - - @Test - public void integrityCheckStatusFailTest() { - Project project = qm.createProject("Acme Application", null, null, null, null, null, true, false); - Component component = new Component(); - component.setProject(project); - component.setName("ABC"); - component.setPurl("pkg:maven/org.acme/abc"); - IntegrityAnalysis integrityAnalysis = new IntegrityAnalysis(); - integrityAnalysis.setComponent(component); - integrityAnalysis.setIntegrityCheckStatus(IntegrityMatchStatus.HASH_MATCH_FAILED); - Date published = new Date(); - integrityAnalysis.setUpdatedAt(published); - integrityAnalysis.setId(component.getId()); - integrityAnalysis.setMd5HashMatchStatus(IntegrityMatchStatus.HASH_MATCH_FAILED); - integrityAnalysis.setSha1HashMatchStatus(HASH_MATCH_UNKNOWN); - integrityAnalysis.setSha256HashMatchStatus(HASH_MATCH_UNKNOWN); - integrityAnalysis.setSha512HashMatchStatus(HASH_MATCH_FAILED); - qm.persist(integrityAnalysis); - component = qm.createComponent(component, false); - Response response = jersey.target(V1_COMPONENT + "/" + component.getUuid() + "/integritycheckstatus") - .request().header(X_API_KEY, apiKey).get(Response.class); - Assert.assertEquals(200, response.getStatus(), 0); - Assert.assertNull(response.getHeaderString(TOTAL_COUNT_HEADER)); - JsonObject json = parseJsonObject(response); - Assert.assertNotNull(json); - Assert.assertEquals(HASH_MATCH_FAILED.name(), json.getString("md5HashMatchStatus")); - Assert.assertEquals(HASH_MATCH_FAILED.name(), json.getString("integrityCheckStatus")); - Assert.assertEquals(HASH_MATCH_FAILED.name(), json.getString("sha512HashMatchStatus")); - Assert.assertEquals(published.toString(), Date.from(Instant.ofEpochSecond(json.getJsonNumber("updatedAt").longValue() / 1000)).toString()); - } - - @Test - public void integrityMetaDataFoundTest() { - Project project = qm.createProject("Acme Application", null, null, null, null, null, true, false); - Component component = new Component(); - component.setProject(project); - component.setName("ABC"); - component.setPurl("pkg:maven/io.micrometer/micrometer-registry-prometheus@1.9.4?type=jar"); - Date published = new Date(); - component = qm.createComponent(component, false); - IntegrityMetaComponent integrityMetaComponent = new IntegrityMetaComponent(); - integrityMetaComponent.setPurl(component.getPurl().toString()); - integrityMetaComponent.setPublishedAt(published); - integrityMetaComponent.setLastFetch(published); - integrityMetaComponent.setStatus(FetchStatus.PROCESSED); - integrityMetaComponent.setRepositoryUrl("https://repo1.maven.org/maven2/io/micrometer/micrometer-registry-prometheus/1.9.4/micrometer-registry-prometheus-1.9.4.jar"); - integrityMetaComponent.setMd5("45e5bdba87362b16852ec279c254eb57"); - integrityMetaComponent.setSha1("45e5bdba87362b16852ec279c254eb57"); - qm.createIntegrityMetaComponent(integrityMetaComponent); - - Response response = jersey.target(V1_COMPONENT + "/integritymetadata") - .queryParam("purl", component.getPurl()) - .request().header(X_API_KEY, apiKey).get(Response.class); - Assert.assertEquals(200, response.getStatus(), 0); - Assert.assertNull(response.getHeaderString(TOTAL_COUNT_HEADER)); - JsonObject json = parseJsonObject(response); - Assert.assertNotNull(json); - Assert.assertEquals("https://repo1.maven.org/maven2/io/micrometer/micrometer-registry-prometheus/1.9.4/micrometer-registry-prometheus-1.9.4.jar", json.getString("repositoryUrl")); - Assert.assertEquals("45e5bdba87362b16852ec279c254eb57", json.getString("md5")); - Assert.assertEquals("45e5bdba87362b16852ec279c254eb57", json.getString("sha1")); - Assert.assertEquals(published.toString(), Date.from(Instant.ofEpochSecond(json.getJsonNumber("publishedAt").longValue() / 1000)).toString()); - } - - @Test - public void integrityMetaDataNotFoundTest() { - Project project = qm.createProject("Acme Application", null, null, null, null, null, true, false); - Component component = new Component(); - component.setProject(project); - component.setName("ABC"); - component.setPurl("pkg:maven/io.micrometer/micrometer-registry-prometheus@1.9.4?type=jar"); - Date published = new Date(); - component = qm.createComponent(component, false); - Response response = jersey.target(V1_COMPONENT + "/integritymetadata") - .queryParam("purl", component.getPurl()) - .request().header(X_API_KEY, apiKey).get(Response.class); - Assert.assertEquals(404, response.getStatus(), 0); - } - - @Test - public void integrityMetaDataInvalidPurlTest() { - Project project = qm.createProject("Acme Application", null, null, null, null, null, true, false); - Component component = new Component(); - component.setProject(project); - component.setName("ABC"); - component.setPurl("pkg:maven/io.micrometer/micrometer-registry-prometheus@1.9.4?type=jar"); - Date published = new Date(); - component = qm.createComponent(component, false); - Response response = jersey.target(V1_COMPONENT + "/integritymetadata") - .queryParam("purl", "component.getPurl()") - .request().header(X_API_KEY, apiKey).get(Response.class); - Assert.assertEquals(400, response.getStatus(), 0); - } - @Test public void getComponentByIdentityWithCoordinatesTest() { final Project projectA = qm.createProject("projectA", null, "1.0", null, null, null, true, false); diff --git a/src/test/java/org/dependencytrack/tasks/BomUploadProcessingTaskTest.java b/src/test/java/org/dependencytrack/tasks/BomUploadProcessingTaskTest.java index 669ae48cd..ec7831231 100644 --- a/src/test/java/org/dependencytrack/tasks/BomUploadProcessingTaskTest.java +++ b/src/test/java/org/dependencytrack/tasks/BomUploadProcessingTaskTest.java @@ -264,7 +264,7 @@ public void informTestWithComponentAlreadyExistsForIntegrityCheck() throws Excep integrityMeta.setPurl("pkg:maven/com.example/xmlutil@1.0.0?download_url=https%3A%2F%2Fon-premises.url%2Frepository%2Fnpm%2F%40babel%2Fhelper-split-export-declaration%2Fhelper-split-export-declaration%2Fhelper-split-export-declaration%2Fhelper-split-export-declaration%2Fhelper-split-export-declaration-7.18.6.tgz"); integrityMeta.setStatus(FetchStatus.IN_PROGRESS); integrityMeta.setLastFetch(Date.from(Instant.now().minus(2, ChronoUnit.HOURS))); - qm.createIntegrityMetaComponent(integrityMeta); + qm.persist(integrityMeta); new BomUploadProcessingTask().inform(bomUploadEvent); assertBomProcessedNotification(); assertThat(kafkaMockProducer.history()).satisfiesExactly( diff --git a/src/test/java/org/dependencytrack/tasks/IntegrityAnalysisTaskTest.java b/src/test/java/org/dependencytrack/tasks/IntegrityAnalysisTaskTest.java deleted file mode 100644 index d77381913..000000000 --- a/src/test/java/org/dependencytrack/tasks/IntegrityAnalysisTaskTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.tasks; - -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.IntegrityAnalysisEvent; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.FetchStatus; -import org.dependencytrack.model.IntegrityAnalysis; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.contrib.java.lang.system.EnvironmentVariables; - -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import java.util.Date; -import java.util.UUID; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.dependencytrack.model.IntegrityMatchStatus.HASH_MATCH_PASSED; - -public class IntegrityAnalysisTaskTest extends PersistenceCapableTest { - - @Rule - public EnvironmentVariables environmentVariables = new EnvironmentVariables(); - - @Before - public void before() throws Exception { - super.before(); - - environmentVariables.set("INTEGRITY_CHECK_ENABLED", "true"); - } - - @Test - public void shouldPerformIntegrityAnalysisIfMetaDataExists() { - // Create an active project with one component. - final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); - final var componentProjectA = new Component(); - UUID uuid = UUID.randomUUID(); - componentProjectA.setProject(projectA); - componentProjectA.setName("acme-lib-a"); - componentProjectA.setVersion("1.0.1"); - componentProjectA.setPurl("pkg:maven/foo/bar@1.2.3"); - componentProjectA.setPurlCoordinates("pkg:maven/foo/bar@1.2.3"); - componentProjectA.setUuid(uuid); - componentProjectA.setMd5("098f6bcd4621d373cade4e832627b4f6"); - componentProjectA.setSha1("a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"); - - Component c = qm.persist(componentProjectA); - - var integrityMetaComponent = new IntegrityMetaComponent(); - integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3"); - integrityMetaComponent.setStatus(FetchStatus.PROCESSED); - integrityMetaComponent.setMd5("098f6bcd4621d373cade4e832627b4f6"); - integrityMetaComponent.setSha1("a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"); - Date date = Date.from(Instant.now().minus(15, ChronoUnit.MINUTES)); - integrityMetaComponent.setLastFetch(date); - IntegrityMetaComponent integrityData = qm.persist(integrityMetaComponent); - - new IntegrityAnalysisTask().inform(new IntegrityAnalysisEvent(c.getUuid(), qm.detach(IntegrityMetaComponent.class, integrityData.getId()))); - IntegrityAnalysis integrityResult = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); - assertThat(integrityResult).isNotNull(); - assertThat(integrityResult.getIntegrityCheckStatus()).isEqualTo(HASH_MATCH_PASSED); - } - - @Test - public void shouldNotPerformAnalysisIfComponentUuidIsMissing() { - // Create an active project with one component. - final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); - final var componentProjectA = new Component(); - UUID uuid = UUID.randomUUID(); - componentProjectA.setProject(projectA); - componentProjectA.setName("acme-lib-a"); - componentProjectA.setVersion("1.0.1"); - componentProjectA.setPurl("pkg:maven/foo/bar@1.2.3"); - componentProjectA.setPurlCoordinates("pkg:maven/foo/bar@1.2.3"); - componentProjectA.setUuid(uuid); - componentProjectA.setMd5("098f6bcd4621d373cade4e832627b4f6"); - componentProjectA.setSha1("a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"); - - Component c = qm.persist(componentProjectA); - - var integrityMetaComponent = new IntegrityMetaComponent(); - integrityMetaComponent.setPurl("pkg:maven/foo/bar@1.2.3"); - integrityMetaComponent.setStatus(FetchStatus.PROCESSED); - integrityMetaComponent.setMd5("098f6bcd4621d373cade4e832627b4f6"); - integrityMetaComponent.setSha1("a94a8fe5ccb19ba61c4c0873d391e987982fbbd3"); - Date date = Date.from(Instant.now().minus(15, ChronoUnit.MINUTES)); - integrityMetaComponent.setLastFetch(date); - IntegrityMetaComponent integrityData = qm.persist(integrityMetaComponent); - - new IntegrityAnalysisTask().inform(new IntegrityAnalysisEvent(null, qm.detach(IntegrityMetaComponent.class, integrityData.getId()))); - IntegrityAnalysis integrityResult = qm.getIntegrityAnalysisByComponentUuid(c.getUuid()); - assertThat(integrityResult).isNull(); - } -} diff --git a/src/test/java/org/dependencytrack/tasks/IntegrityMetaInitializerTaskTest.java b/src/test/java/org/dependencytrack/tasks/IntegrityMetaInitializerTaskTest.java deleted file mode 100644 index a9da28052..000000000 --- a/src/test/java/org/dependencytrack/tasks/IntegrityMetaInitializerTaskTest.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * This file is part of Dependency-Track. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - * SPDX-License-Identifier: Apache-2.0 - * Copyright (c) OWASP Foundation. All Rights Reserved. - */ -package org.dependencytrack.tasks; - -import org.dependencytrack.PersistenceCapableTest; -import org.dependencytrack.event.IntegrityMetaInitializerEvent; -import org.dependencytrack.event.kafka.KafkaTopics; -import org.dependencytrack.model.Component; -import org.dependencytrack.model.IntegrityMetaComponent; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.contrib.java.lang.system.EnvironmentVariables; - -import java.sql.Date; -import java.time.Instant; -import java.time.temporal.ChronoUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatNoException; -import static org.dependencytrack.model.FetchStatus.IN_PROGRESS; - -public class IntegrityMetaInitializerTaskTest extends PersistenceCapableTest { - - @Rule - public EnvironmentVariables environmentVariables = new EnvironmentVariables(); - - final Component componentPersisted = new Component(); - - @Before - public void persistComponentData() { - environmentVariables.set("INTEGRITY_INITIALIZER_ENABLED", "true"); - final var projectA = qm.createProject("acme-app-a", null, "1.0.0", null, null, null, true, false); - componentPersisted.setProject(projectA); - componentPersisted.setName("acme-lib-a"); - componentPersisted.setInternal(false); - componentPersisted.setPurlCoordinates("pkg:maven/acme/acme-lib-a@1.0.1"); - componentPersisted.setPurl("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); - qm.persist(componentPersisted); - kafkaMockProducer.clear(); - } - - @Test - public void testIntegrityMetaInitializer() { - final var IntegrityMetaComponent = new IntegrityMetaComponent(); - IntegrityMetaComponent.setPurl("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); - qm.persist(IntegrityMetaComponent); - new IntegrityMetaInitializerTask().inform(new IntegrityMetaInitializerEvent()); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - assertThat(kafkaMockProducer.history()).satisfiesExactly( - record -> assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()) - ); - } - - @Test - public void shouldNotDispatchEventIfPackageTypeIsNotSupported() { - final var IntegrityMetaComponent = new IntegrityMetaComponent(); - IntegrityMetaComponent.setPurl("pkg:golang/github.com/prometheus/client_model@0.2.0?type=module"); - qm.persist(IntegrityMetaComponent); - new IntegrityMetaInitializerTask().inform(new IntegrityMetaInitializerEvent()); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - assertThat(kafkaMockProducer.history()).isEmpty(); - } - - @Test - public void testIntegrityMetaInitializerWithExistingDataFetchedNotRecently() { - // data exists in IntegrityMetaComponent but last fetched 3 hours ago > 1 hour wait time - var integrityMetaExisting = new IntegrityMetaComponent(); - integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); - integrityMetaExisting.setStatus(IN_PROGRESS); - integrityMetaExisting.setLastFetch(Date.from(Instant.now().minus(3, ChronoUnit.HOURS))); - qm.persist(integrityMetaExisting); - new IntegrityMetaInitializerTask().inform(new IntegrityMetaInitializerEvent()); - // kafka event is dispatched - assertThat(kafkaMockProducer.history().size()).isEqualTo(1); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - } - - @Test - public void testIntegrityMetaInitializerWithExistingDataNotProcessed() { - // data exists in IntegrityMetaComponent but not processed yet - var integrityMetaExisting = new IntegrityMetaComponent(); - integrityMetaExisting.setPurl(componentPersisted.getPurl().toString()); - qm.persist(integrityMetaExisting); - new IntegrityMetaInitializerTask().inform(new IntegrityMetaInitializerEvent()); - // kafka event is dispatched - assertThat(kafkaMockProducer.history().size()).isEqualTo(1); - assertThat(qm.getIntegrityMetaComponentCount()).isEqualTo(1); - } - - @Test - public void testIntegrityMetaInitializerWithNonExistentComponent() { - // Create a meta component. - var integrityMetaExisting = new IntegrityMetaComponent(); - integrityMetaExisting.setPurl("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); - qm.persist(integrityMetaExisting); - - // Delete the component such that the meta component's PURL no longer matches - // any record in the COMPONENT table. - qm.delete(componentPersisted); - - // No exception must be raised. - assertThatNoException().isThrownBy(() -> new IntegrityMetaInitializerTask().inform(new IntegrityMetaInitializerEvent())); - - // No Kafka record must be sent. - assertThat(kafkaMockProducer.history().size()).isZero(); - } - -} diff --git a/src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTaskTest.java b/src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTaskTest.java index 6a5e2b64c..8ee7adfc9 100644 --- a/src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTaskTest.java +++ b/src/test/java/org/dependencytrack/tasks/RepositoryMetaAnalyzerTaskTest.java @@ -93,21 +93,33 @@ record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_ record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), // projectC record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), // projectD record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), // projectE + // NB: Components are dispatched in ascending order by PURL. record -> { + // componentProjectA assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + assertThat(record.key()).asString().isEqualTo("pkg:maven/acme/acme-lib-a"); final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); - assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1"); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); + assertThat(command.getComponent().getInternal()).isFalse(); + }, + record -> { + // componentProjectD + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + assertThat(record.key()).asString().isEqualTo("pkg:maven/acme/acme-lib-a"); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1?qux=quux"); assertThat(command.getComponent().getInternal()).isFalse(); }, - // componentProjectB must not have been submitted, because it does not have a PURL - // componentProjectC must not have been submitted, because it belongs to an inactive project - // componentProjectD has the same PURL coordinates as componentProjectA and is not submitted again record -> { + // componentProjectE assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + assertThat(record.key()).asString().isEqualTo("pkg:maven/acme/acme-lib-a"); final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); - assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1"); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1?fizz=buzz"); assertThat(command.getComponent().getInternal()).isTrue(); } + // componentProjectB must not have been submitted, because it does not have a PURL + // componentProjectC must not have been submitted, because it belongs to an inactive project ); } @@ -160,26 +172,40 @@ public void testProjectRepositoryMetaAnalysis() { assertThat(kafkaMockProducer.history()).satisfiesExactlyInAnyOrder( record -> assertThat(record.topic()).isEqualTo(KafkaTopics.NOTIFICATION_PROJECT_CREATED.name()), + // NB: Components are dispatched in ascending order by PURL. record -> { + // componentA assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + assertThat(record.key()).asString().isEqualTo("pkg:maven/acme/acme-lib-a"); final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); - assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1"); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1?foo=bar"); assertThat(command.getComponent().getInternal()).isFalse(); }, record -> { + // componentE assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + assertThat(record.key()).asString().isEqualTo("pkg:maven/acme/acme-lib-a"); final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); - assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1"); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1?qux=quux"); assertThat(command.getComponent().getInternal()).isTrue(); }, - // componentB must not have been submitted, because it does not have a PURL record -> { + // componentD + assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + assertThat(record.key()).asString().isEqualTo("pkg:maven/acme/acme-lib-a"); + final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-a@1.0.1?qux=quux"); + assertThat(command.getComponent().getInternal()).isFalse(); + }, + record -> { + // componentC assertThat(record.topic()).isEqualTo(KafkaTopics.REPO_META_ANALYSIS_COMMAND.name()); + assertThat(record.key()).asString().isEqualTo("pkg:maven/acme/acme-lib-c"); final var command = deserializeValue(KafkaTopics.REPO_META_ANALYSIS_COMMAND, record); - assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-c@3.0.1"); + assertThat(command.getComponent().getPurl()).isEqualTo("pkg:maven/acme/acme-lib-c@3.0.1?foo=bar"); assertThat(command.getComponent().getInternal()).isFalse(); } - // componentD has the same PURL coordinates as componentA nad is not submitted again + // componentB must not have been submitted, because it does not have a PURL ); }