Merge pull request #1277 from TG1999/add_importer_specific_improver

Add importer specific improver

CHANGELOG.rst

@@ -2,6 +2,13 @@ Release notes
 =============
 
 
+Version v33.4.0
+----------------
+
+- We added importer specific improvers and removed default improver
+  additionally improve recent advisories first.
+
+
 Version v33.3.0
 ----------------
 

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = vulnerablecode
-version = 33.3.0
+version = 33.4.0
 license = Apache-2.0 AND CC-BY-SA-4.0
 
 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390

vulnerabilities/importers/apache_tomcat.py

@@ -138,7 +138,7 @@ def fetch_advisory_links(self, url):
         for tag in soup.find_all("a"):
             link = tag.get("href")
 
-            if "security-" in link and any(char.isdigit() for char in link):
+            if link and "security-" in link and any(char.isdigit() for char in link):
                 yield urllib.parse.urljoin(url, link)
 
     def advisory_data(self):

vulnerabilities/improvers/__init__.py

@@ -7,11 +7,38 @@
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
 
-from vulnerabilities.improvers import default
+from vulnerabilities.improvers import importer_specific_improver
 from vulnerabilities.improvers import valid_versions
 
 IMPROVERS_REGISTRY = [
-    default.DefaultImprover,
+    importer_specific_improver.NVDImprover,
+    importer_specific_improver.DebianImprover,
+    importer_specific_improver.DebianOvalImprover,
+    importer_specific_improver.AlpineLinuxImprover,
+    importer_specific_improver.ApacheHTTPDImprover,
+    importer_specific_improver.ApacheKafkaImprover,
+    importer_specific_improver.ApacheTomcatImprover,
+    importer_specific_improver.ArchLinuxImprover,
+    importer_specific_improver.ElixirSecurityImprover,
+    importer_specific_improver.FireEyeImprover,
+    importer_specific_improver.GentooImprover,
+    importer_specific_improver.GitHubAPIImprover,
+    importer_specific_improver.GitLabAPIImprover,
+    importer_specific_improver.IstioImprover,
+    importer_specific_improver.MozillaImprover,
+    importer_specific_improver.NginxImprover,
+    importer_specific_improver.NpmImprover,
+    importer_specific_improver.OpensslImprover,
+    importer_specific_improver.PostgreSQLImprover,
+    importer_specific_improver.ProjectKBMSRImprover,
+    importer_specific_improver.PyPaImprover,
+    importer_specific_improver.PyPIImprover,
+    importer_specific_improver.RedhatImprover,
+    importer_specific_improver.RetireDotnetImprover,
+    importer_specific_improver.SUSESeverityScoreImprover,
+    importer_specific_improver.UbuntuImprover,
+    importer_specific_improver.UbuntuUSNImprover,
+    importer_specific_improver.XenImprover,
     valid_versions.NginxBasicImprover,
     valid_versions.ApacheHTTPDImprover,
     valid_versions.DebianBasicImprover,

vulnerabilities/improvers/default.py

@@ -12,11 +12,13 @@
 from typing import List
 from typing import Tuple
 
+from django.db.models import Q
 from django.db.models.query import QuerySet
 from packageurl import PackageURL
 
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackage
+from vulnerabilities.importer import Importer
 from vulnerabilities.improver import MAX_CONFIDENCE
 from vulnerabilities.improver import Improver
 from vulnerabilities.improver import Inference
@@ -34,10 +36,17 @@ class DefaultImprover(Improver):
     information source.
     """
 
+    importer: Importer
+
     @property
     def interesting_advisories(self) -> QuerySet:
-        for advisory in Advisory.objects.all().paginated():
-            yield advisory
+        if hasattr(self, "importer"):
+            return (
+                Advisory.objects.filter(Q(created_by=self.importer.qualified_name))
+                .order_by("-date_collected")
+                .paginated()
+            )
+        return Advisory.objects.all().order_by("-date_collected").paginated()
 
     def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]:
         if not advisory_data:

vulnerabilities/improvers/importer_specific_improver.py

@@ -0,0 +1,150 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from vulnerabilities.importers.alpine_linux import AlpineImporter
+from vulnerabilities.importers.apache_httpd import ApacheHTTPDImporter
+from vulnerabilities.importers.apache_kafka import ApacheKafkaImporter
+from vulnerabilities.importers.apache_tomcat import ApacheTomcatImporter
+from vulnerabilities.importers.archlinux import ArchlinuxImporter
+from vulnerabilities.importers.debian import DebianImporter
+from vulnerabilities.importers.debian_oval import DebianOvalImporter
+from vulnerabilities.importers.elixir_security import ElixirSecurityImporter
+from vulnerabilities.importers.fireeye import FireyeImporter
+from vulnerabilities.importers.gentoo import GentooImporter
+from vulnerabilities.importers.github import GitHubAPIImporter
+from vulnerabilities.importers.gitlab import GitLabAPIImporter
+from vulnerabilities.importers.istio import IstioImporter
+from vulnerabilities.importers.mozilla import MozillaImporter
+from vulnerabilities.importers.nginx import NginxImporter
+from vulnerabilities.importers.npm import NpmImporter
+from vulnerabilities.importers.nvd import NVDImporter
+from vulnerabilities.importers.openssl import OpensslImporter
+from vulnerabilities.importers.postgresql import PostgreSQLImporter
+from vulnerabilities.importers.project_kb_msr2019 import ProjectKBMSRImporter
+from vulnerabilities.importers.pypa import PyPaImporter
+from vulnerabilities.importers.pysec import PyPIImporter
+from vulnerabilities.importers.redhat import RedhatImporter
+from vulnerabilities.importers.retiredotnet import RetireDotnetImporter
+from vulnerabilities.importers.suse_scores import SUSESeverityScoreImporter
+from vulnerabilities.importers.ubuntu import UbuntuImporter
+from vulnerabilities.importers.ubuntu_usn import UbuntuUSNImporter
+from vulnerabilities.importers.xen import XenImporter
+from vulnerabilities.improvers.default import DefaultImprover
+
+
+class NVDImprover(DefaultImprover):
+    importer = NVDImporter
+
+
+class AlpineLinuxImprover(DefaultImprover):
+    importer = AlpineImporter
+
+
+class ApacheHTTPDImprover(DefaultImprover):
+    importer = ApacheHTTPDImporter
+
+
+class ApacheKafkaImprover(DefaultImprover):
+    importer = ApacheKafkaImporter
+
+
+class ApacheTomcatImprover(DefaultImprover):
+    importer = ApacheTomcatImporter
+
+
+class ArchLinuxImprover(DefaultImprover):
+    importer = ArchlinuxImporter
+
+
+class DebianImprover(DefaultImprover):
+    importer = DebianImporter
+
+
+class DebianOvalImprover(DefaultImprover):
+    importer = DebianOvalImporter
+
+
+class ElixirSecurityImprover(DefaultImprover):
+    importer = ElixirSecurityImporter
+
+
+class FireEyeImprover(DefaultImprover):
+    importer = FireyeImporter
+
+
+class GentooImprover(DefaultImprover):
+    importer = GentooImporter
+
+
+class GitHubAPIImprover(DefaultImprover):
+    importer = GitHubAPIImporter
+
+
+class GitLabAPIImprover(DefaultImprover):
+    importer = GitLabAPIImporter
+
+
+class IstioImprover(DefaultImprover):
+    importer = IstioImporter
+
+
+class MozillaImprover(DefaultImprover):
+    importer = MozillaImporter
+
+
+class NginxImprover(DefaultImprover):
+    importer = NginxImporter
+
+
+class NpmImprover(DefaultImprover):
+    importer = NpmImporter
+
+
+class OpensslImprover(DefaultImprover):
+    importer = OpensslImporter
+
+
+class PostgreSQLImprover(DefaultImprover):
+    importer = PostgreSQLImporter
+
+
+class ProjectKBMSRImprover(DefaultImprover):
+    importer = ProjectKBMSRImporter
+
+
+class PyPaImprover(DefaultImprover):
+    importer = PyPaImporter
+
+
+class PyPIImprover(DefaultImprover):
+    importer = PyPIImporter
+
+
+class RedhatImprover(DefaultImprover):
+    importer = RedhatImporter
+
+
+class RetireDotnetImprover(DefaultImprover):
+    importer = RetireDotnetImporter
+
+
+class SUSESeverityScoreImprover(DefaultImprover):
+    importer = SUSESeverityScoreImporter
+
+
+class UbuntuImprover(DefaultImprover):
+    importer = UbuntuImporter
+
+
+class UbuntuUSNImprover(DefaultImprover):
+    importer = UbuntuUSNImporter
+
+
+class XenImprover(DefaultImprover):
+    importer = XenImporter

vulnerabilities/tests/test_importer_specific_improver.py

@@ -0,0 +1,31 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import datetime
+
+import pytest
+
+from vulnerabilities.importers.nvd import NVDImporter
+from vulnerabilities.improve_runner import ImproveRunner
+from vulnerabilities.improvers.importer_specific_improver import NVDImprover
+from vulnerabilities.models import Advisory
+from vulnerabilities.models import Alias
+
+
+@pytest.mark.django_db
+def test_improvement_of_importer_specific_advisories():
+    Advisory.objects.create(
+        aliases=["CVE-2021-22"],
+        summary="TEST",
+        created_by=NVDImporter.qualified_name,
+        date_collected=datetime.datetime.now(tz=datetime.timezone.utc),
+    )
+    ImproveRunner(NVDImprover).run()
+    alias = Alias.objects.filter(alias="CVE-2021-22").first()
+    assert alias is not None

vulnerablecode/__init__.py

@@ -12,7 +12,7 @@
 import warnings
 from pathlib import Path
 
-__version__ = "33.3.0"
+__version__ = "33.4.0"
 
 
 def command_line():