Merge pull request #985 from nexB/969-migrate-postgresql-importer

Migrate postgresql importer
aboutcode-org · Nov 21, 2022 · 469d20e · 469d20e
2 parents bf49673 + 1aca009
commit 469d20e
Show file tree

Hide file tree

Showing 8 changed files with 288 additions and 225 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -3,6 +3,12 @@ Release notes
 
 
 
+Version v30.3.2
+----------------
+
+- We re-enabled support for the PostgreSQL securities advisories importer.
+
+
 Version v30.3.1
 ----------------
 
@@ -17,10 +23,10 @@ Version v30.3.0
 This is a feature update release including minor bug fixes and the introduction
 of API keys and API throttling.
 
-- We enabled API throttling for a basic user and for a staff user 
+- We enabled API throttling for a basic user and for a staff user
   they can have unlimited access on API.
 
-- We added throttle rate for each API endpoint and it can be 
+- We added throttle rate for each API endpoint and it can be
   configured from the settings #991 https://github.com/nexB/vulnerablecode/issues/991
 
 - We improved how we import NVD data

diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py
@@ -16,6 +16,7 @@
 from vulnerabilities.importers import nginx
 from vulnerabilities.importers import nvd
 from vulnerabilities.importers import openssl
+from vulnerabilities.importers import postgresql
 from vulnerabilities.importers import pypa
 from vulnerabilities.importers import pysec
 from vulnerabilities.importers import redhat
@@ -31,6 +32,7 @@
     pysec.PyPIImporter,
     debian.DebianImporter,
     gitlab.GitLabAPIImporter,
+    postgresql.PostgreSQLImporter,
     pypa.PyPaImporter,
     archlinux.ArchlinuxImporter,
     ubuntu.UbuntuImporter,

diff --git a/vulnerabilities/importers/postgresql.py b/vulnerabilities/importers/postgresql.py
@@ -12,35 +12,40 @@
 import requests
 from bs4 import BeautifulSoup
 from packageurl import PackageURL
+from univers.version_range import GenericVersionRange
+from univers.versions import GenericVersion
 
 from vulnerabilities import severity_systems
 from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import AffectedPackage
 from vulnerabilities.importer import Importer
 from vulnerabilities.importer import Reference
 from vulnerabilities.importer import VulnerabilitySeverity
-from vulnerabilities.utils import nearest_patched_package
 
 
 class PostgreSQLImporter(Importer):
 
     root_url = "https://www.postgresql.org/support/security/"
+    license_url = "https://www.postgresql.org/about/licence/"
+    spdx_license_expression = "PostgreSQL"
 
-    def updated_advisories(self):
-        advisories = []
+    def advisory_data(self):
         known_urls = {self.root_url}
         visited_urls = set()
+        data_by_url = {}
         while True:
             unvisited_urls = known_urls - visited_urls
             for url in unvisited_urls:
                 data = requests.get(url).content
-                advisories.extend(to_advisories(data))
+                data_by_url[url] = data
                 visited_urls.add(url)
                 known_urls.update(find_advisory_urls(data))
 
             if known_urls == visited_urls:
                 break
 
-        return self.batch_advisories(advisories)
+        for url, data in data_by_url.items():
+            yield from to_advisories(data)
 
 
 def to_advisories(data):
@@ -54,29 +59,43 @@ def to_advisories(data):
         if "windows" in summary.lower():
             pkg_qualifiers = {"os": "windows"}
 
-        affected_packages = [
-            PackageURL(
-                type="generic",
-                name="postgresql",
-                version=version.strip(),
-                qualifiers=pkg_qualifiers,
-            )
-            for version in affected_col.text.split(",")
-        ]
-
-        fixed_packages = [
-            PackageURL(
-                type="generic",
-                name="postgresql",
-                version=version.strip(),
-                qualifiers=pkg_qualifiers,
+        affected_packages = []
+        affected_version_list = affected_col.text.split(",")
+        fixed_version_list = fixed_col.text.split(",")
+
+        if fixed_version_list:
+            for fixed_version in fixed_version_list:
+                affected_packages.append(
+                    AffectedPackage(
+                        package=PackageURL(
+                            name="postgresql",
+                            # TODO: See https://github.com/nexB/vulnerablecode/issues/990
+                            type="generic",
+                            qualifiers=pkg_qualifiers,
+                        ),
+                        affected_version_range=GenericVersionRange.from_versions(
+                            affected_version_list
+                        )
+                        if affected_version_list
+                        else None,
+                        fixed_version=GenericVersion(fixed_version) if fixed_version else None,
+                    )
+                )
+        elif affected_version_list:
+            affected_packages.append(
+                AffectedPackage(
+                    package=PackageURL(
+                        name="postgresql",
+                        # TODO: See https://github.com/nexB/vulnerablecode/issues/990
+                        type="generic",
+                        qualifiers=pkg_qualifiers,
+                    ),
+                    affected_version_range=GenericVersionRange.from_versions(affected_version_list),
+                )
             )
-            for version in fixed_col.text.split(",")
-            if version
-        ]
-
+        cve_id = ""
         try:
-            cve_id = ref_col.select("nobr")[0].text
+            cve_id = ref_col.select(".nobr")[0].text
             # This is for the anomaly in https://www.postgresql.org/support/security/8.1/ 's
             # last entry
         except IndexError:
@@ -102,21 +121,20 @@ def to_advisories(data):
                     )
                     severities.append(severity)
             references.append(Reference(url=link, severities=severities))
-
-        advisories.append(
-            AdvisoryData(
-                vulnerability_id=cve_id,
-                summary=summary,
-                references=references,
-                affected_packages=nearest_patched_package(affected_packages, fixed_packages),
+        if cve_id:
+            advisories.append(
+                AdvisoryData(
+                    aliases=[cve_id],
+                    summary=summary,
+                    references=references,
+                    affected_packages=affected_packages,
+                )
             )
-        )
-
     return advisories
 
 
 def find_advisory_urls(page_data):
-    soup = BeautifulSoup(page_data)
+    soup = BeautifulSoup(page_data, features="lxml")
     return {
         urlparse.urljoin("https://www.postgresql.org/", a_tag.attrs["href"])
         for a_tag in soup.select("h3+ p a")

diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py
@@ -317,14 +317,40 @@ class Meta:
         ordering = ["vulnerability", "reference"]
 
 
+def purl_to_dict(purl: PackageURL):
+    """
+    Return a dict of purl components suitable for use in a queryset.
+    We need to have specific empty values for using in querysets because of our peculiar model structure.
+
+    For example::
+    >>> purl_to_dict(PackageURL.from_string("pkg:generic/postgres"))
+    {'type': 'generic', 'namespace': '', 'name': 'postgres', 'version': '', 'qualifiers': {}, 'subpath': ''}
+    >>> purl_to_dict(PackageURL.from_string("pkg:generic/postgres/[email protected]?foo=bar#baz"))
+    {'type': 'generic', 'namespace': 'postgres', 'name': 'postgres', 'version': '1.2', 'qualifiers': {'foo': 'bar'}, 'subpath': 'baz'}
+    """
+    if isinstance(purl, str):
+        purl = PackageURL.from_string(purl)
+
+    return dict(
+        type=purl.type,
+        namespace=purl.namespace or "",
+        name=purl.name,
+        version=purl.version or "",
+        qualifiers=purl.qualifiers or {},
+        subpath=purl.subpath or "",
+    )
+
+
 class PackageQuerySet(BaseQuerySet, PackageURLQuerySet):
     def get_or_create_from_purl(self, purl: PackageURL):
         """
         Return an existing or new Package (created if neeed) given a
         ``purl`` PackageURL.
         """
-        purl_fields = without_empty_values(purl.to_dict(encode=True))
-        package, _ = Package.objects.get_or_create(**purl_fields)
+        if isinstance(purl, str):
+            purl = PackageURL.from_string(purl)
+
+        package, _ = Package.objects.get_or_create(**purl_to_dict(purl=purl))
         return package
 
     def for_package_url_object(self, purl):
@@ -333,15 +359,12 @@ def for_package_url_object(self, purl):
         ``purl`` string is validated and transformed into filtering lookups. If
         this is a PackageURL object it is reused as-is.
         """
-        if isinstance(purl, PackageURL):
-            lookups = without_empty_values(purl.to_dict(encode=True))
-            return self.filter(**lookups)
-
-        elif isinstance(purl, str):
-            return self.for_package_url(purl, encode=False)
-
-        else:
+        if not purl:
             return self.none()
+        if isinstance(purl, str):
+            purl = PackageURL.from_string(purl)
+        lookups = without_empty_values(purl.to_dict(encode=True))
+        return self.filter(**lookups)
 
     def affected(self):
         """

diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py
@@ -37,7 +37,6 @@ def no_rmtree(monkeypatch):
     "test_msr2019.py",
     "test_npm.py",
     "test_package_managers.py",
-    "test_postgresql.py",
     "test_retiredotnet.py",
     "test_ruby.py",
     "test_rust.py",

diff --git a/vulnerabilities/tests/test_data/postgresql/advisories.html b/vulnerabilities/tests/test_data/postgresql/advisories.html
@@ -109,7 +109,7 @@ <h2>Known security issues in all supported versions</h2>
 
     <tr>
       <td>
-        <nobr><a href="/support/security/CVE-2020-10733/">CVE-2020-10733</a></nobr><br>
+        <a href="/support/security/CVE-2020-10733/" class="nobr">CVE-2020-10733</a><br>
         <a href="/about/news/postgresql-123-118-1013-9618-and-9522-released-2038/">Announcement</a><br>
       </td>
       <td>12, 11, 10, 9.6</td>
@@ -122,7 +122,7 @@ <h2>Known security issues in all supported versions</h2>
 
       <tr>
         <td>
-          <nobr><a href="/support/security/CVE-2020-1720/">CVE-2020-1720</a></nobr><br>
+          <a href="/support/security/CVE-2020-1720/" class="nobr">CVE-2020-1720</a><br>
           <a href="/about/news/postgresql-122-117-1012-9617-9521-and-9426-released-2011/">Announcement</a><br>
         </td>
         <td>12, 11, 10, 9.6</td>