Merge branch 'main' into data-export

requirements.txt

@@ -106,7 +106,7 @@ toml==0.10.2
 tomli==2.0.1
 traitlets==5.1.1
 typing_extensions==4.1.1
-univers==30.11.0
+univers==30.12.0
 urllib3==1.26.19
 wcwidth==0.2.5
 websocket-client==0.59.0

setup.cfg

@@ -71,7 +71,7 @@ install_requires =
 
     #essentials
     packageurl-python>=0.10.5rc1
-    univers>=30.11.0
+    univers>=30.12.0
     license-expression>=21.6.14
 
     # file and data formats

vulnerabilities/api.py

@@ -47,7 +47,7 @@ class VulnerabilityReferenceSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = VulnerabilityReference
-        fields = ["reference_url", "reference_id", "scores", "url"]
+        fields = ["reference_url", "reference_id", "reference_type", "scores", "url"]
 
 
 class BaseResourceSerializer(serializers.HyperlinkedModelSerializer):

vulnerabilities/importer.py

@@ -76,6 +76,7 @@ def from_dict(cls, severity: dict):
 @dataclasses.dataclass(order=True)
 class Reference:
     reference_id: str = ""
+    reference_type: str = ""
     url: str = ""
     severities: List[VulnerabilitySeverity] = dataclasses.field(default_factory=list)
 
@@ -85,11 +86,17 @@ def __post_init__(self):
 
     def normalized(self):
         severities = sorted(self.severities)
-        return Reference(reference_id=self.reference_id, url=self.url, severities=severities)
+        return Reference(
+            reference_id=self.reference_id,
+            url=self.url,
+            severities=severities,
+            reference_type=self.reference_type,
+        )
 
     def to_dict(self):
         return {
             "reference_id": self.reference_id,
+            "reference_type": self.reference_type,
             "url": self.url,
             "severities": [severity.to_dict() for severity in self.severities],
         }
@@ -98,6 +105,7 @@ def to_dict(self):
     def from_dict(cls, ref: dict):
         return cls(
             reference_id=ref["reference_id"],
+            reference_type=ref["reference_type"],
             url=ref["url"],
             severities=[
                 VulnerabilitySeverity.from_dict(severity) for severity in ref["severities"]

vulnerabilities/importers/fireeye.py

@@ -89,9 +89,9 @@ def get_references(references):
     """
     Return a list of Reference from a list of URL reference in md format
     >>> get_references(["- http://1-4a.com/cgi-bin/alienform/af.cgi"])
-    [Reference(reference_id='', url='http://1-4a.com/cgi-bin/alienform/af.cgi', severities=[])]
+    [Reference(reference_id='', reference_type='', url='http://1-4a.com/cgi-bin/alienform/af.cgi', severities=[])]
     >>> get_references(["- [Mitre CVE-2021-42712](https://www.cve.org/CVERecord?id=CVE-2021-42712)"])
-    [Reference(reference_id='', url='https://www.cve.org/CVERecord?id=CVE-2021-42712', severities=[])]
+    [Reference(reference_id='', reference_type='', url='https://www.cve.org/CVERecord?id=CVE-2021-42712', severities=[])]
     """
     urls = []
     for ref in references:

vulnerabilities/importers/github.py

@@ -37,6 +37,7 @@
     "PIP": "pypi",
     "RUBYGEMS": "gem",
     "NPM": "npm",
+    "RUST": "cargo",
     # "GO": "golang",
 }
 
@@ -47,7 +48,7 @@
 # TODO: We will try to gather more info from GH API
 # Check https://github.com/nexB/vulnerablecode/issues/1039#issuecomment-1366458885
 # Check https://github.com/nexB/vulnerablecode/issues/645
-# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM'}
+# set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM', 'RUST'}
 # second '%s' is interesting, it will have the value '' for the first request,
 GRAPHQL_QUERY_TEMPLATE = """
 query{
@@ -139,7 +140,7 @@ def get_purl(pkg_type: str, github_name: str) -> Optional[PackageURL]:
         vendor, _, name = github_name.partition("/")
         return PackageURL(type=pkg_type, namespace=vendor, name=name)
 
-    if pkg_type in ("nuget", "pypi", "gem", "golang", "npm"):
+    if pkg_type in ("nuget", "pypi", "gem", "golang", "npm", "cargo"):
         return PackageURL(type=pkg_type, name=github_name)
 
     logger.error(f"get_purl: Unknown package type {pkg_type}")

vulnerabilities/improve_runner.py

@@ -98,12 +98,14 @@ def process_inferences(
 
             reference = VulnerabilityReference.objects.get_or_none(
                 reference_id=ref.reference_id,
+                reference_type=ref.reference_type,
                 url=ref.url,
             )
 
             if not reference:
                 reference = create_valid_vulnerability_reference(
                     reference_id=ref.reference_id,
+                    reference_type=ref.reference_type,
                     url=ref.url,
                 )
                 if not reference:
@@ -167,14 +169,15 @@ def process_inferences(
     return inferences_processed_count
 
 
-def create_valid_vulnerability_reference(url, reference_id=None):
+def create_valid_vulnerability_reference(url, reference_type="", reference_id=None):
     """
     Create and return a new validated VulnerabilityReference from a
     ``url`` and ``reference_id``.
     Return None and log a warning if this is not a valid reference.
     """
     reference = VulnerabilityReference(
         reference_id=reference_id,
+        reference_type=reference_type,
         url=url,
     )
 

vulnerabilities/migrations/0058_alter_vulnerabilityreference_options_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.1.13 on 2024-08-01 22:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0057_kev"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="vulnerabilityreference",
+            options={"ordering": ["reference_id", "url", "reference_type"]},
+        ),
+        migrations.AddField(
+            model_name="vulnerabilityreference",
+            name="reference_type",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("advisory", "Advisory"),
+                    ("exploit", "Exploit"),
+                    ("mailing_list", "Mailing List"),
+                    ("bug", "Bug"),
+                    ("other", "Other"),
+                ],
+                max_length=20,
+            ),
+        ),
+    ]

vulnerabilities/models.py

@@ -359,6 +359,22 @@ class VulnerabilityReference(models.Model):
         unique=True,
     )
 
+    ADVISORY = "advisory"
+    EXPLOIT = "exploit"
+    MAILING_LIST = "mailing_list"
+    BUG = "bug"
+    OTHER = "other"
+
+    REFERENCE_TYPES = [
+        (ADVISORY, "Advisory"),
+        (EXPLOIT, "Exploit"),
+        (MAILING_LIST, "Mailing List"),
+        (BUG, "Bug"),
+        (OTHER, "Other"),
+    ]
+
+    reference_type = models.CharField(max_length=20, choices=REFERENCE_TYPES, blank=True)
+
     reference_id = models.CharField(
         max_length=200,
         help_text="An optional reference ID, such as DSA-4465-1 when available",
@@ -368,7 +384,7 @@ class VulnerabilityReference(models.Model):
     objects = VulnerabilityReferenceQuerySet.as_manager()
 
     class Meta:
-        ordering = ["reference_id", "url"]
+        ordering = ["reference_id", "url", "reference_type"]
 
     def __str__(self):
         reference_id = f" {self.reference_id}" if self.reference_id else ""

vulnerabilities/templates/packages.html

@@ -48,7 +48,7 @@
                             <span
                                 class="has-tooltip-multiline has-tooltip-black has-tooltip-arrow has-tooltip-text-left"
                                 data-tooltip="This is the number of vulnerabilities fixed by the package.">
-                                <span class="affected-fixed">Fixed by</span> vulnerabilities
+                                <span class="affected-fixed">Fixing</span> vulnerabilities
                             </span>
                         </th>
                     </tr>

vulnerabilities/templates/vulnerability_details.html

@@ -244,6 +244,7 @@
                     <thead>
                         <tr>
                             <th style="width: 250px;"> Reference id </th>
+                            <th style="width: 250px;"> Reference type </th>
                             <th> URL </th>
                         </tr>
                     </thead>
@@ -254,6 +255,13 @@
                         {% else %}
                         <td></td>
                         {% endif %}
+
+                        {% if ref.reference_type %}
+                        <td class="wrap-strings">{{ ref.get_reference_type_display }}</td>
+                        {% else %}
+                        <td></td>
+                        {% endif %}
+
                         <td class="wrap-strings"><a href="{{ ref.url }}" target="_blank">{{ ref.url }}<i
                                     class="fa fa-external-link fa_link_custom"></i></a></td>
                     </tr>

vulnerabilities/tests/test_api.py

@@ -21,6 +21,7 @@
 
 from vulnerabilities.api import MinimalPackageSerializer
 from vulnerabilities.api import PackageSerializer
+from vulnerabilities.api import VulnerabilityReferenceSerializer
 from vulnerabilities.models import Alias
 from vulnerabilities.models import ApiUser
 from vulnerabilities.models import Package
@@ -161,6 +162,9 @@ def setUp(self):
             namespace="ubuntu",
             qualifiers={"distro": "jessie"},
         )
+        self.ref = VulnerabilityReference.objects.create(
+            reference_type="advisory", reference_id="CVE-xxx-xxx", url="https://example.com"
+        )
         self.user = ApiUser.objects.create_api_user(username="[email protected]")
         self.auth = f"Token {self.user.auth_token.key}"
         self.client = APIClient(enforce_csrf_checks=True)
@@ -181,6 +185,16 @@ def test_package_serializer(self):
         purls = {r["purl"] for r in response}
         self.assertIn("pkg:deb/ubuntu/[email protected]?distro=jessie", purls)
 
+    def test_vulnerability_reference_serializer(self):
+        response = VulnerabilityReferenceSerializer(instance=self.ref).data
+        assert response == {
+            "reference_url": "https://example.com",
+            "reference_id": "CVE-xxx-xxx",
+            "reference_type": "advisory",
+            "scores": [],
+            "url": "https://example.com",
+        }
+
 
 class APITestCaseVulnerability(TransactionTestCase):
     def setUp(self):

vulnerabilities/tests/test_data/apache_httpd/CVE-1999-1199-apache-httpd-expected.json

@@ -20,6 +20,7 @@
   "references": [
     {
       "reference_id": "CVE-1999-1199",
+      "reference_type": "",
       "url": "https://httpd.apache.org/security/json/CVE-1999-1199.json",
       "severities": [
         {

vulnerabilities/tests/test_data/apache_httpd/CVE-2017-9798-apache-httpd-expected.json

@@ -20,6 +20,7 @@
   "references": [
     {
       "reference_id": "CVE-2017-9798",
+      "reference_type": "",
       "url": "https://httpd.apache.org/security/json/CVE-2017-9798.json",
       "severities": [
         {

vulnerabilities/tests/test_data/apache_httpd/CVE-2021-44224-apache-httpd-expected.json

@@ -20,6 +20,7 @@
   "references": [
     {
       "reference_id": "CVE-2021-44224",
+      "reference_type": "",
       "url": "https://httpd.apache.org/security/json/CVE-2021-44224.json",
       "severities": [
         {

vulnerabilities/tests/test_data/apache_httpd/CVE-2022-28614-apache-httpd-expected.json

@@ -20,6 +20,7 @@
   "references": [
     {
       "reference_id": "CVE-2022-28614",
+      "reference_type": "",
       "url": "https://httpd.apache.org/security/json/CVE-2022-28614.json",
       "severities": [
         {

vulnerabilities/tests/test_data/apache_httpd/apache-httpd-improver-expected.json

@@ -43,6 +43,7 @@
     "references": [
       {
         "reference_id": "CVE-2021-44224",
+        "reference_type": "",
         "url": "https://httpd.apache.org/security/json/CVE-2021-44224.json",
         "severities": [
           {
@@ -91,6 +92,7 @@
     "references": [
       {
         "reference_id": "CVE-2021-44224",
+        "reference_type": "",
         "url": "https://httpd.apache.org/security/json/CVE-2021-44224.json",
         "severities": [
           {