Refactor BOM upload processing for better efficiency, correctness, an…

…d consistency Ports DependencyTrack/dependency-track#3357 from Dependency-Track v4.11.0 The majority of changes already existed in the Hyades codebase, since they were originally ported from Hyades to DT :) The following changes were NOT ported: * `IndexingInstanceLifecycleListener`: Lucene was removed in Hyades, keeping the index current is not a concern * `L2CacheEvictingInstanceLifecycleListener`: The L2 cache has been disabled globally in Hyades, invalidating L2 cache entries is not a concern * The `ConfigKey.BOM_PROCESSING_TASK_V2_ENABLED`: The legacy BOM processing task no longer exists in Hyades, hence making this config useless Signed-off-by: nscuro <[email protected]>
DependencyTrack · Jun 10, 2024 · 339cb94 · 339cb94
1 parent b2952a5
commit 339cb94
Show file tree

Hide file tree

Showing 16 changed files with 941 additions and 847 deletions.
diff --git a/src/main/java/org/dependencytrack/common/MdcKeys.java b/src/main/java/org/dependencytrack/common/MdcKeys.java
@@ -23,12 +23,19 @@
  */
 public final class MdcKeys {
 
+    public static final String MDC_BOM_FORMAT = "bomFormat";
+    public static final String MDC_BOM_SERIAL_NUMBER = "bomSerialNumber";
+    public static final String MDC_BOM_SPEC_VERSION = "bomSpecVersion";
+    public static final String MDC_BOM_UPLOAD_TOKEN = "bomUploadToken";
+    public static final String MDC_BOM_VERSION = "bomVersion";
     public static final String MDC_COMPONENT_UUID = "componentUuid";
     public static final String MDC_KAFKA_RECORD_TOPIC = "kafkaRecordTopic";
     public static final String MDC_KAFKA_RECORD_PARTITION = "kafkaRecordPartition";
     public static final String MDC_KAFKA_RECORD_OFFSET = "kafkaRecordOffset";
     public static final String MDC_KAFKA_RECORD_KEY = "kafkaRecordKey";
+    public static final String MDC_PROJECT_NAME = "projectName";
     public static final String MDC_PROJECT_UUID = "projectUuid";
+    public static final String MDC_PROJECT_VERSION = "projectVersion";
     public static final String MDC_SCAN_TOKEN = "scanToken";
 
     private MdcKeys() {

diff --git a/src/main/java/org/dependencytrack/event/IntegrityAnalysisEvent.java b/src/main/java/org/dependencytrack/event/IntegrityAnalysisEvent.java
@@ -23,13 +23,15 @@
 
 import java.util.UUID;
 
-public class IntegrityAnalysisEvent implements Event {
+import static org.dependencytrack.util.PersistenceUtil.assertNonPersistent;
 
-    private UUID uuid;
+public class IntegrityAnalysisEvent implements Event {
 
-    private IntegrityMetaComponent integrityMetaComponent;
+    private final UUID uuid;
+    private final IntegrityMetaComponent integrityMetaComponent;
 
     public IntegrityAnalysisEvent(UUID uuid, IntegrityMetaComponent integrityMetaComponent) {
+        assertNonPersistent(integrityMetaComponent, "integrityMetaComponent must not be persistent");
         this.uuid = uuid;
         this.integrityMetaComponent = integrityMetaComponent;
     }

diff --git a/src/main/java/org/dependencytrack/model/License.java b/src/main/java/org/dependencytrack/model/License.java
@@ -92,6 +92,7 @@ public enum FetchGroup {
     }
 
     private static final long serialVersionUID = -1707920279688859358L;
+    public static final License UNRESOLVED = new License();
 
     @PrimaryKey
     @Persistent(valueStrategy = IdGeneratorStrategy.NATIVE)

diff --git a/src/main/java/org/dependencytrack/model/Project.java b/src/main/java/org/dependencytrack/model/Project.java
@@ -300,6 +300,8 @@ public enum FetchGroup {
     @ApiModelProperty(accessMode = ApiModelProperty.AccessMode.READ_ONLY)
     private ProjectMetadata metadata;
 
+    private transient String bomRef;
+
     private transient ProjectMetrics metrics;
 
     private transient List<ProjectVersion> versions;
@@ -511,6 +513,14 @@ public void setActive(Boolean active) {
         this.active = active;
     }
 
+    public String getBomRef() {
+        return bomRef;
+    }
+
+    public void setBomRef(String bomRef) {
+        this.bomRef = bomRef;
+    }
+
     public ProjectMetrics getMetrics() {
         return metrics;
     }

diff --git a/src/main/java/org/dependencytrack/parser/cyclonedx/util/ModelConverter.java b/src/main/java/org/dependencytrack/parser/cyclonedx/util/ModelConverter.java
@@ -21,7 +21,10 @@
 import alpine.common.logging.Logger;
 import com.github.packageurl.MalformedPackageURLException;
 import com.github.packageurl.PackageURL;
+import org.apache.commons.collections4.MultiValuedMap;
+import org.apache.commons.collections4.multimap.HashSetValuedHashMap;
 import org.apache.commons.lang3.StringUtils;
+import org.cyclonedx.model.BomReference;
 import org.cyclonedx.model.Dependency;
 import org.cyclonedx.model.Hash;
 import org.cyclonedx.model.LicenseChoice;
@@ -129,15 +132,16 @@ public static Project convertToProject(final org.cyclonedx.model.Metadata cdxMet
 
     public static Project convertToProject(final org.cyclonedx.model.Component cdxComponent) {
         final var project = new Project();
+        project.setBomRef(useOrGenerateRandomBomRef(cdxComponent.getBomRef()));
         project.setAuthor(trimToNull(cdxComponent.getAuthor()));
         project.setPublisher(trimToNull(cdxComponent.getPublisher()));
+        project.setSupplier(convert(cdxComponent.getSupplier()));
         project.setClassifier(convertClassifier(cdxComponent.getType()).orElse(Classifier.APPLICATION));
         project.setGroup(trimToNull(cdxComponent.getGroup()));
         project.setName(trimToNull(cdxComponent.getName()));
         project.setVersion(trimToNull(cdxComponent.getVersion()));
         project.setDescription(trimToNull(cdxComponent.getDescription()));
         project.setExternalReferences(convertExternalReferences(cdxComponent.getExternalReferences()));
-        project.setSupplier(ModelConverter.convert(cdxComponent.getSupplier()));
 
         if (cdxComponent.getPurl() != null) {
             try {
@@ -165,6 +169,7 @@ public static List<Component> convertComponents(final List<org.cyclonedx.model.C
 
     public static Component convertComponent(final org.cyclonedx.model.Component cdxComponent) {
         final var component = new Component();
+        component.setBomRef(useOrGenerateRandomBomRef(cdxComponent.getBomRef()));
         component.setAuthor(trimToNull(cdxComponent.getAuthor()));
         component.setPublisher(trimToNull(cdxComponent.getPublisher()));
         component.setSupplier(convert(cdxComponent.getSupplier()));
@@ -389,7 +394,7 @@ public static List<ServiceComponent> convertServices(final List<org.cyclonedx.mo
 
     public static ServiceComponent convertService(final org.cyclonedx.model.Service cdxService) {
         final var service = new ServiceComponent();
-        service.setBomRef(trimToNull(cdxService.getBomRef()));
+        service.setBomRef(useOrGenerateRandomBomRef(cdxService.getBomRef()));
         service.setProvider(convert(cdxService.getProvider()));
         service.setGroup(trimToNull(cdxService.getGroup()));
         service.setName(trimToNull(cdxService.getName()));
@@ -418,6 +423,25 @@ public static ServiceComponent convertService(final org.cyclonedx.model.Service
         return service;
     }
 
+    public static MultiValuedMap<String, String> convertDependencyGraph(final List<Dependency> cdxDependencies) {
+        final var dependencyGraph = new HashSetValuedHashMap<String, String>();
+        if (cdxDependencies == null || cdxDependencies.isEmpty()) {
+            return dependencyGraph;
+        }
+
+        for (final Dependency cdxDependency : cdxDependencies) {
+            if (cdxDependency.getDependencies() == null || cdxDependency.getDependencies().isEmpty()) {
+                continue;
+            }
+
+            final List<String> directDependencies = cdxDependency.getDependencies().stream()
+                    .map(BomReference::getRef).toList();
+            dependencyGraph.putAll(cdxDependency.getRef(), directDependencies);
+        }
+
+        return dependencyGraph;
+    }
+
     private static Optional<Classifier> convertClassifier(final org.cyclonedx.model.Component.Type cdxComponentType) {
         return Optional.ofNullable(cdxComponentType)
                 .map(Enum::name)
@@ -482,6 +506,12 @@ private static List<DataClassification> convertDataClassification(final List<org
                 .toList();
     }
 
+    private static String useOrGenerateRandomBomRef(final String bomRef) {
+        return Optional.ofNullable(bomRef)
+                .map(StringUtils::trimToNull)
+                .orElseGet(() -> UUID.randomUUID().toString());
+    }
+
     public static <T> List<T> flatten(final Collection<T> items,
                                       final Function<T, Collection<T>> childrenGetter,
                                       final BiConsumer<T, Collection<T>> childrenSetter) {

diff --git a/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java b/src/main/java/org/dependencytrack/persistence/ComponentQueryManager.java
@@ -656,65 +656,47 @@ public void recursivelyDelete(Component component, boolean commitIndex) {
     /**
      * Returns a component by matching its identity information.
      * <p>
-     * Note that this method employs a stricter matching logic than {@link #matchIdentity(Project, ComponentIdentity)}
-     * and {@link #matchIdentity(ComponentIdentity)}. For example, if {@code purl} of the given {@link ComponentIdentity}
-     * is {@code null}, this method will use a query that explicitly checks for the {@code purl} column to be {@code null}.
+     * Note that this method employs a stricter matching logic than {@link #matchIdentity(Project, ComponentIdentity)}.
+     * For example, if {@code purl} of the given {@link ComponentIdentity} is {@code null}, this method will use a
+     * query that explicitly checks for the {@code purl} column to be {@code null}.
      * Whereas other methods will simply not include {@code purl} in the query in such cases.
      *
      * @param project the Project the component is a dependency of
      * @param cid     the identity values of the component
      * @return a Component object, or null if not found
+     * @since 4.11.0
      */
-    public Component matchSingleIdentity(final Project project, final ComponentIdentity cid) {
-        var filterParts = new ArrayList<String>();
-        final var params = new HashMap<String, Object>();
-
-        if (cid.getPurl() != null) {
-            filterParts.add("(purl != null && purl == :purl)");
-            params.put("purl", cid.getPurl().canonicalize());
-        } else {
-            filterParts.add("purl == null");
-        }
-
-        if (cid.getCpe() != null) {
-            filterParts.add("(cpe != null && cpe == :cpe)");
-            params.put("cpe", cid.getCpe());
-        } else {
-            filterParts.add("cpe == null");
-        }
-
-        if (cid.getSwidTagId() != null) {
-            filterParts.add("(swidTagId != null && swidTagId == :swidTagId)");
-            params.put("swidTagId", cid.getSwidTagId());
-        } else {
-            filterParts.add("swidTagId == null");
-        }
-
-        var coordinatesFilter = "(";
-        if (cid.getGroup() != null) {
-            coordinatesFilter += "group == :group";
-            params.put("group", cid.getGroup());
-        } else {
-            coordinatesFilter += "group == null";
-        }
-        coordinatesFilter += " && name == :name";
-        params.put("name", cid.getName());
-        if (cid.getVersion() != null) {
-            coordinatesFilter += " && version == :version";
-            params.put("version", cid.getVersion());
-        } else {
-            coordinatesFilter += " && version == null";
+    public Component matchSingleIdentityExact(final Project project, final ComponentIdentity cid) {
+        final Pair<String, Map<String, Object>> queryFilterParamsPair = buildExactComponentIdentityQuery(project, cid);
+        final Query<Component> query = pm.newQuery(Component.class, queryFilterParamsPair.getKey());
+        query.setNamedParameters(queryFilterParamsPair.getRight());
+        try {
+            return query.executeUnique();
+        } finally {
+            query.closeAll();
         }
-        coordinatesFilter += ")";
-        filterParts.add(coordinatesFilter);
-
-        final var filter = "project == :project && (" + String.join(" && ", filterParts) + ")";
-        params.put("project", project);
+    }
 
-        final Query<Component> query = pm.newQuery(Component.class, filter);
-        query.setNamedParameters(params);
+    /**
+     * Returns the first component matching a given {@link ComponentIdentity} in a {@link Project}.
+     *
+     * @param project the Project the component is a dependency of
+     * @param cid     the identity values of the component
+     * @return a Component object, or null if not found
+     * @since 4.11.0
+     */
+    public Component matchFirstIdentityExact(final Project project, final ComponentIdentity cid) {
+        final Pair<String, Map<String, Object>> queryFilterParamsPair = buildExactComponentIdentityQuery(project, cid);
+        final Query<Component> query = pm.newQuery(Component.class, queryFilterParamsPair.getKey());
+        query.setNamedParameters(queryFilterParamsPair.getRight());
+        query.setRange(0, 1);
         try {
-            return query.executeUnique();
+            final List<Component> result = query.executeList();
+            if (result.isEmpty()) {
+                return null;
+            }
+
+            return result.getFirst();
         } finally {
             query.closeAll();
         }
@@ -810,6 +792,55 @@ private static Pair<String, Map<String, Object>> buildComponentIdentityQuery(fin
         return Pair.of(filter, params);
     }
 
+    private static Pair<String, Map<String, Object>> buildExactComponentIdentityQuery(final Project project, final ComponentIdentity cid) {
+        var filterParts = new ArrayList<String>();
+        final var params = new HashMap<String, Object>();
+
+        if (cid.getPurl() != null) {
+            filterParts.add("(purl != null && purl == :purl)");
+            params.put("purl", cid.getPurl().canonicalize());
+        } else {
+            filterParts.add("purl == null");
+        }
+
+        if (cid.getCpe() != null) {
+            filterParts.add("(cpe != null && cpe == :cpe)");
+            params.put("cpe", cid.getCpe());
+        } else {
+            filterParts.add("cpe == null");
+        }
+
+        if (cid.getSwidTagId() != null) {
+            filterParts.add("(swidTagId != null && swidTagId == :swidTagId)");
+            params.put("swidTagId", cid.getSwidTagId());
+        } else {
+            filterParts.add("swidTagId == null");
+        }
+
+        var coordinatesFilter = "(";
+        if (cid.getGroup() != null) {
+            coordinatesFilter += "group == :group";
+            params.put("group", cid.getGroup());
+        } else {
+            coordinatesFilter += "group == null";
+        }
+        coordinatesFilter += " && name == :name";
+        params.put("name", cid.getName());
+        if (cid.getVersion() != null) {
+            coordinatesFilter += " && version == :version";
+            params.put("version", cid.getVersion());
+        } else {
+            coordinatesFilter += " && version == null";
+        }
+        coordinatesFilter += ")";
+        filterParts.add(coordinatesFilter);
+
+        final var filter = "project == :project && (" + String.join(" && ", filterParts) + ")";
+        params.put("project", project);
+
+        return Pair.of(filter, params);
+    }
+
     /**
      * Intelligently adds dependencies for components that are not already a dependency
      * of the specified project and removes the dependency relationship for components

diff --git a/src/main/java/org/dependencytrack/persistence/QueryManager.java b/src/main/java/org/dependencytrack/persistence/QueryManager.java
@@ -983,8 +983,12 @@ public List<VulnerableSoftware> getAllVulnerableSoftware(final String cpePart, f
         return getVulnerableSoftwareQueryManager().getAllVulnerableSoftware(cpePart, cpeVendor, cpeProduct, purl);
     }
 
-    public Component matchSingleIdentity(final Project project, final ComponentIdentity cid) {
-        return getComponentQueryManager().matchSingleIdentity(project, cid);
+    public Component matchSingleIdentityExact(final Project project, final ComponentIdentity cid) {
+        return getComponentQueryManager().matchSingleIdentityExact(project, cid);
+    }
+
+    public Component matchFirstIdentityExact(final Project project, final ComponentIdentity cid) {
+        return getComponentQueryManager().matchFirstIdentityExact(project, cid);
     }
 
     public List<Component> matchIdentity(final Project project, final ComponentIdentity cid) {

diff --git a/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java b/src/main/java/org/dependencytrack/resources/v1/ComponentResource.java
@@ -382,7 +382,7 @@ public Response createComponent(@PathParam("uuid") String uuid, Component jsonCo
             component.setClassifier(jsonComponent.getClassifier());
             component.setPurl(jsonComponent.getPurl());
             component.setPurlCoordinates(PurlUtil.silentPurlCoordinatesOnly(jsonComponent.getPurl()));
-            component.setInternal(InternalComponentIdentificationUtil.isInternalComponent(component, qm));
+            component.setInternal(InternalComponentIdentificationUtil.isInternalComponent(component));
             component.setCpe(StringUtils.trimToNull(jsonComponent.getCpe()));
             component.setSwidTagId(StringUtils.trimToNull(jsonComponent.getSwidTagId()));
             component.setCopyright(StringUtils.trimToNull(jsonComponent.getCopyright()));
@@ -489,7 +489,7 @@ public Response updateComponent(Component jsonComponent) {
                 component.setClassifier(jsonComponent.getClassifier());
                 component.setPurl(jsonComponent.getPurl());
                 component.setPurlCoordinates(PurlUtil.silentPurlCoordinatesOnly(component.getPurl()));
-                component.setInternal(InternalComponentIdentificationUtil.isInternalComponent(component, qm));
+                component.setInternal(InternalComponentIdentificationUtil.isInternalComponent(component));
                 component.setCpe(StringUtils.trimToNull(jsonComponent.getCpe()));
                 component.setSwidTagId(StringUtils.trimToNull(jsonComponent.getSwidTagId()));
                 component.setCopyright(StringUtils.trimToNull(jsonComponent.getCopyright()));