From 1e4079ddb78caa9630e2dff75fe84a0a80f44d58 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 25 Jul 2023 20:53:57 -0700 Subject: [PATCH 01/53] Add initial fixed-affected-matching work #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 12 ++++ .../templates/package_details.html | 63 ++++++++++++++++++- vulnerabilities/views.py | 1 - 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 16c93002f..545953316 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -588,6 +588,18 @@ def affected_by(self): # legacy aliases vulnerable_to = affected_by + @property + def test_get_fixing_purls(self): + """ + This is a test -- the goal is to display the closest fixing version for a PURL that is greater + than the affected version and is the same type. We want to filter on type, namespace, + name, qualifiers and subpath for the affected PURL. + """ + return [ + abc.fixed_by_packages + for abc in self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) + ] + @property # TODO: consider renaming to "fixes" or "fixing" ? (TBD) and updating the docstring def fixing(self): diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 6a391d3d3..bb076c930 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -40,6 +40,58 @@ +
+ Let's try to display fixing packages for this package: {{ package.purl }} +
+ package.purl = {{ package.purl }} +
+
+ package.qualifiers = {{ package.qualifiers }} +
+
+ package.vulnerabilities = {{ package.vulnerabilities }} +
+
+ package.package_url = {{ package.package_url }} +
+
+ package.plain_package_url = {{ package.plain_package_url }} +
+
+ package.purl_object = {{ package.purl_object }} +
+ +
+ package.fixing = {{ package.fixing }} +
+
+ package.fixed_packages = Server Error (500) +
+
+ package.is_vulnerable = {{ package.is_vulnerable }} +
+
+ package.get_absolute_url = {{ package.get_absolute_url }} +
+ +
+ package.affected_by = {{ package.affected_by }} +
+
+ package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} +
+ + {% for abc in package.affected_by %} + +
{{ abc }} -- {{ abc.fixed_by_packages }}
+
{{ abc }} -- {% for pkg in abc.fixed_by_packages %}{{ pkg.purl }}{% endfor %}
+ + {% endfor %} + + + +
+
Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }}) @@ -51,6 +103,7 @@ Vulnerability Summary Aliases + Test fixing PURLs @@ -74,10 +127,18 @@ {% endif %} {% endfor %} + + + + {% for pkg in vulnerability.fixed_by_packages %} + {{ pkg.purl }} + {% endfor %} + {% empty %} - + + This package is not known to be affected by vulnerabilities. diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index e96f43a6d..fad42eae8 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -176,7 +176,6 @@ class ApiUserCreateView(generic.CreateView): template_name = "api_user_creation_form.html" def form_valid(self, form): - try: response = super().form_valid(form) except ValidationError: From 624f047cb21f04de0fbad927ccdd6c7e7ae8bbd2 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Fri, 28 Jul 2023 09:26:40 -0700 Subject: [PATCH 02/53] Explore context and Package class approaches for affected-fixed package matching #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Note that my updated code is still in testing/dev stage and has not yet been completed or cleaned. Signed-off-by: John M. Horan --- vulnerabilities/models.py | 80 +++++++++-- .../templates/package_details.html | 134 +++++++++++++++--- vulnerabilities/views.py | 35 +++++ 3 files changed, 221 insertions(+), 28 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 545953316..dbdf45ab8 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -22,6 +22,7 @@ from django.core.validators import MinValueValidator from django.db import models from django.db.models import Count +from django.db.models import Prefetch from django.db.models import Q from django.db.models.functions import Length from django.db.models.functions import Trim @@ -588,18 +589,6 @@ def affected_by(self): # legacy aliases vulnerable_to = affected_by - @property - def test_get_fixing_purls(self): - """ - This is a test -- the goal is to display the closest fixing version for a PURL that is greater - than the affected version and is the same type. We want to filter on type, namespace, - name, qualifiers and subpath for the affected PURL. - """ - return [ - abc.fixed_by_packages - for abc in self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - ] - @property # TODO: consider renaming to "fixes" or "fixing" ? (TBD) and updating the docstring def fixing(self): @@ -631,6 +620,73 @@ def get_absolute_url(self): """ return reverse("package_details", args=[self.purl]) + def get_fixed_packages(self, package): + """ + Return a queryset of all packages that fix a vulnerability with + same type, namespace, name, subpath and qualifiers of the `package` + """ + return Package.objects.filter( + name=package.name, + namespace=package.namespace, + type=package.type, + qualifiers=package.qualifiers, + subpath=package.subpath, + packagerelatedvulnerability__fix=True, + ).distinct() + + @property + def test_get_fixing_purls(self): + """ + This is a test -- the goal is to display the closest fixing version for a PURL that is greater + than the affected version and is the same type. We want to filter on type, namespace, + name, qualifiers and subpath for the affected PURL. + """ + + fixed_packages = self.get_fixed_packages(package=self) + + # Prefetch: + + # Where do we get "fix" from? + # qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=fix) + + # Not clear to me how we use this: + # qs = qs.prefetch_related( + # Prefetch( + # "packages", + # queryset=fixed_packages, + # to_attr="filtered_fixed_packages", + # ) + # ) + + matching_fixed_packages = [] + + # closest_subsequent_fixed_package = [] + + test_dict = {"affected_purl": self.purl} + test_dict["vulnerabilities"] = [] + + for vuln in self.affected_by: + test_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + + for fixing_pkg in vuln.fixed_by_packages: + # Do not add "backports". TODO: Do we need to use univers to compare versions? + if fixing_pkg in fixed_packages and fixing_pkg.version > self.version: + matching_fixed_packages.append(fixing_pkg) + + # TODO: We also need to check if there is more than one fixing package and if so, keep only the package closest to the affect package's version. + # TODO: Do we want to check whether the fixing version has any vulnerabilities of its own? + + # ======================================================== + print("\ntest_dict = \n{}\n".format(test_dict)) + print(json.dumps(test_dict, indent=4)) + # ======================================================== + + return matching_fixed_packages + + +# 2023-07-27 Thursday 09:50:45. JMH: check this out re related api.py code +# TODO: Not clear how this is involved. + class PackageRelatedVulnerability(models.Model): """ diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index bb076c930..63c0fc37e 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -43,52 +43,131 @@
Let's try to display fixing packages for this package: {{ package.purl }}
- package.purl = {{ package.purl }} + package.purl = {{ package.purl }}
- package.qualifiers = {{ package.qualifiers }} + package.qualifiers = {{ package.qualifiers }}
- package.vulnerabilities = {{ package.vulnerabilities }} + package.vulnerabilities = {{ package.vulnerabilities }}
- package.package_url = {{ package.package_url }} + package.package_url = {{ package.package_url }}
- package.plain_package_url = {{ package.plain_package_url }} + package.plain_package_url = {{ package.plain_package_url }}
- package.purl_object = {{ package.purl_object }} + package.purl_object = {{ package.purl_object }}
- package.fixing = {{ package.fixing }} + package.fixing = {{ package.fixing }}
- package.fixed_packages = Server Error (500) + package.fixed_packages = Server Error (500)
- package.is_vulnerable = {{ package.is_vulnerable }} + package.is_vulnerable = {{ package.is_vulnerable }}
- package.get_absolute_url = {{ package.get_absolute_url }} + package.get_absolute_url = {{ package.get_absolute_url }}
- package.affected_by = {{ package.affected_by }} + package.affected_by = {{ package.affected_by }}
+ +
+ package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} +
+
- package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} + package.test_get_fixing_purls = {{ package.test_get_fixing_purls }}
{% for abc in package.affected_by %} -
{{ abc }} -- {{ abc.fixed_by_packages }}
-
{{ abc }} -- {% for pkg in abc.fixed_by_packages %}{{ pkg.purl }}{% endfor %}
+
{{ abc }} -- {{ abc.fixed_by_packages }}
+
{{ abc }} -- {% for pkg in abc.fixed_by_packages %}{{ pkg.purl }}{% endfor %}
+ + {% endfor %} + +
abc.test_matching_fixed_by_packages
+ {% for abc in package.affected_by %} + + + +
{{ abc }} -- + {% for pkg in abc.test_matching_fixed_by_packages %} +
{{ pkg.purl }}
+ {% endfor %} +
{% endfor %} +
+
test01
+ {% for thing in test01 %} + {{ thing }} +
+ {% endfor %} +
+ +
+
test02
+ {% for thing in test02 %} + - {{ thing }} +
+ {% endfor %} +
+ +
+
test03
+ {% for thing in test03 %} + - {{ thing }} +
+ {% for stuff in thing.test_matching_fixed_by_packages %} + ==> {{ stuff }} +
+ {% endfor %} +
+ {% endfor %} +
+ +
+
test04
+ {% for thing in test04 %} + - {{ thing }} +
+ {% endfor %} +
+ +
+ +
context["test_get_fixing_purls"]
+ {% for thing in test_get_fixing_purls %} + - {{ thing }} +
+ {% endfor %} + {% for vulnerability in affected_by_vulnerabilities %} +
+ {{ vulnerability }} +
+
+ {% for pkg in vulnerability.fixed_by_packages %} + {% if pkg in test_get_fixing_purls %} +
+ {{ pkg.purl }} +
+ {% endif %} + {% endfor %} +
+ {% endfor %} +
@@ -103,7 +182,7 @@ Vulnerability Summary Aliases - Test fixing PURLs + Fixing Packages @@ -131,8 +210,31 @@ {% for pkg in vulnerability.fixed_by_packages %} - {{ pkg.purl }} +
+ {{ pkg.purl }} +
+ {% endfor %} + +
+ + {% for pkg in vulnerability.fixed_by_packages %} + {% if pkg in test04 %} +
+ {{ pkg.purl }} +
+ {% endif %} + {% endfor %} + +
+ + {% for pkg in vulnerability.fixed_by_packages %} + {% if pkg in test_get_fixing_purls %} +
+ {{ pkg.purl }} +
+ {% endif %} {% endfor %} + {% empty %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index fad42eae8..eb181ff93 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -83,6 +83,41 @@ def get_context_data(self, **kwargs): context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) + context["test01"] = [ + "aaa", + "bbb", + "ccc", + ] + context["test02"] = [ + package, + package.type, + package.namespace, + package.name, + package.version, + package.qualifiers, + package.subpath, + ] + context["test03"] = package.affected_by + + # ======================================================== + context["test04"] = [ + fixing_pkg + for vuln in package.affected_by + # for fixing_pkg in vuln.test_matching_fixed_by_packages + for fixing_pkg in vuln.fixed_by_packages + if fixing_pkg.type == package.type + and fixing_pkg.namespace == package.namespace + and fixing_pkg.name == package.name + and fixing_pkg.qualifiers == package.qualifiers + and fixing_pkg.subpath == package.subpath + # I think this version comparison requires the use of univers. + # Plus we just need the closest one, not all that are greater than. + and fixing_pkg.version > package.version + ] + # ======================================================== + context["test_get_fixing_purls"] = package.test_get_fixing_purls + # ======================================================== + return context def get_object(self, queryset=None): From 945b8112bd5a9d7768d336bb6b554800ca1297b4 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Sat, 29 Jul 2023 18:00:26 -0700 Subject: [PATCH 03/53] Add Prefetch and univers-based version comparison #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 115 +++++++++---- .../templates/package_details.html | 155 +----------------- vulnerabilities/views.py | 35 +--- 3 files changed, 86 insertions(+), 219 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index dbdf45ab8..dde000813 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -32,6 +32,7 @@ from packageurl.contrib.django.models import PackageURLQuerySet from packageurl.contrib.django.models import without_empty_values from rest_framework.authtoken.models import Token +from univers import versions from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage @@ -634,58 +635,100 @@ def get_fixed_packages(self, package): packagerelatedvulnerability__fix=True, ).distinct() - @property - def test_get_fixing_purls(self): + def assign_univers_version(self, fixing_pkg): """ - This is a test -- the goal is to display the closest fixing version for a PURL that is greater - than the affected version and is the same type. We want to filter on type, namespace, - name, qualifiers and subpath for the affected PURL. + Identify which univers version applies to the two packages to be compared (self and a fixing package), + evaluate whether the fixing_pkg version is > than the target affected package, and + return True or False. """ - fixed_packages = self.get_fixed_packages(package=self) + # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. - # Prefetch: + # Many more to be added. + match_type_to_univers_version = { + "conan": versions.ConanVersion, + "maven": versions.MavenVersion, + } - # Where do we get "fix" from? - # qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=fix) + command_name = "" - # Not clear to me how we use this: - # qs = qs.prefetch_related( - # Prefetch( - # "packages", - # queryset=fixed_packages, - # to_attr="filtered_fixed_packages", - # ) - # ) + type_to_version = match_type_to_univers_version.get(fixing_pkg.type) + if type_to_version: + print("\t--------------------------") + print("\ttype_to_version = {}\n".format(type_to_version)) + command_name = type_to_version - matching_fixed_packages = [] + else: + print("\ttype_to_version = NO MATCH\n") + command_name = versions.Version - # closest_subsequent_fixed_package = [] + if command_name(fixing_pkg.version) > command_name(self.version): + return True + else: + return False - test_dict = {"affected_purl": self.purl} - test_dict["vulnerabilities"] = [] + @property + def get_fixing_packages(self): + """ + This function identifies the closest fixing version that is greater than the affected version and + is the same type, namespace, name, qualifiers and subpath as the affected package. + """ - for vuln in self.affected_by: - test_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + print("\nself = {}\n".format(self)) - for fixing_pkg in vuln.fixed_by_packages: - # Do not add "backports". TODO: Do we need to use univers to compare versions? - if fixing_pkg in fixed_packages and fixing_pkg.version > self.version: - matching_fixed_packages.append(fixing_pkg) + # This returns all fixing packages that match the target package (type etc.), regardless of fixed vuln. + fixed_packages = self.get_fixed_packages(package=self) - # TODO: We also need to check if there is more than one fixing package and if so, keep only the package closest to the affect package's version. - # TODO: Do we want to check whether the fixing version has any vulnerabilities of its own? + # This returns a list of the vulnerabilities that affect this package (i.e., self). + qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - # ======================================================== - print("\ntest_dict = \n{}\n".format(test_dict)) - print(json.dumps(test_dict, indent=4)) - # ======================================================== + # This takes the list of vulns affecting the current package, retrieves a list of the fixing packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages`. + # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + qs = qs.prefetch_related( + Prefetch( + "packages", + queryset=fixed_packages, + to_attr="filtered_fixed_packages", + ) + ) - return matching_fixed_packages + # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). + print("qs = {}\n".format(qs)) + prefetch_fixed_packages = [] -# 2023-07-27 Thursday 09:50:45. JMH: check this out re related api.py code -# TODO: Not clear how this is involved. + vuln_count = 0 + for vuln in qs: + print("vuln = {}\n".format(vuln)) + print( + "\tqs[vuln_count].filtered_fixed_packages = {}".format( + qs[vuln_count].filtered_fixed_packages + ) + ) + print("") + + # Check the Prefetch qs. + # TODO: Do we want to check whether the fixing version has any vulnerabilities of its own? + for fixing_pkg in qs[vuln_count].filtered_fixed_packages: + print("\tfixing_pkg = {}".format(fixing_pkg)) + print("\tfixing_pkg.type = {}".format(fixing_pkg.type)) + print("\tfixing_pkg.version = {}".format(fixing_pkg.version)) + print("\t--------------------------") + print("\tself.type = {}".format(self.type)) + print("\tself.version = {}".format(self.version)) + + # Assign univers version and compare: False = fixing_pkg.version < self.version (affected version). + # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. + immediate_fix = self.assign_univers_version(fixing_pkg) + print("\t--------------------------") + print("\timmediate_fix = {}\n".format(immediate_fix)) + + if fixing_pkg in fixed_packages and immediate_fix: + prefetch_fixed_packages.append(fixing_pkg) + + vuln_count += 1 + + return prefetch_fixed_packages class PackageRelatedVulnerability(models.Model): diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 63c0fc37e..cfbfdd706 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -40,138 +40,8 @@
-
- Let's try to display fixing packages for this package: {{ package.purl }} -
- package.purl = {{ package.purl }} -
-
- package.qualifiers = {{ package.qualifiers }} -
-
- package.vulnerabilities = {{ package.vulnerabilities }} -
-
- package.package_url = {{ package.package_url }} -
-
- package.plain_package_url = {{ package.plain_package_url }} -
-
- package.purl_object = {{ package.purl_object }} -
- -
- package.fixing = {{ package.fixing }} -
-
- package.fixed_packages = Server Error (500) -
-
- package.is_vulnerable = {{ package.is_vulnerable }} -
-
- package.get_absolute_url = {{ package.get_absolute_url }} -
- -
- package.affected_by = {{ package.affected_by }} -
- -
- package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} -
- -
- package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} -
- - {% for abc in package.affected_by %} - -
{{ abc }} -- {{ abc.fixed_by_packages }}
-
{{ abc }} -- {% for pkg in abc.fixed_by_packages %}{{ pkg.purl }}{% endfor %}
- - {% endfor %} - -
abc.test_matching_fixed_by_packages
- {% for abc in package.affected_by %} - - - -
{{ abc }} -- - {% for pkg in abc.test_matching_fixed_by_packages %} -
{{ pkg.purl }}
- {% endfor %} -
- - {% endfor %} - -
-
test01
- {% for thing in test01 %} - {{ thing }} -
- {% endfor %} -
- -
-
test02
- {% for thing in test02 %} - - {{ thing }} -
- {% endfor %} -
- -
-
test03
- {% for thing in test03 %} - - {{ thing }} -
- {% for stuff in thing.test_matching_fixed_by_packages %} - ==> {{ stuff }} -
- {% endfor %} -
- {% endfor %} -
- -
-
test04
- {% for thing in test04 %} - - {{ thing }} -
- {% endfor %} -
- -
- -
context["test_get_fixing_purls"]
- {% for thing in test_get_fixing_purls %} - - {{ thing }} -
- {% endfor %} - - {% for vulnerability in affected_by_vulnerabilities %} -
- {{ vulnerability }} -
-
- {% for pkg in vulnerability.fixed_by_packages %} - {% if pkg in test_get_fixing_purls %} -
- {{ pkg.purl }} -
- {% endif %} - {% endfor %} -
- {% endfor %} -
- -
-
+
During testing, red = all packages that fix the vulnerability (for pkg in vulnerability.fixed_by_packages) and green = the closest fixing package whose version is greater than the affected package's version (if pkg in get_fixing_packages).
Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }})
@@ -182,7 +52,7 @@ Vulnerability Summary Aliases - Fixing Packages + Fixing Packages @@ -207,39 +77,26 @@ {% endfor %} - - {% for pkg in vulnerability.fixed_by_packages %} -
+
{{ pkg.purl }}
{% endfor %} -
+
{% for pkg in vulnerability.fixed_by_packages %} - {% if pkg in test04 %} -
+ {% if pkg in get_fixing_packages %} +
{{ pkg.purl }}
{% endif %} {% endfor %} -
- - {% for pkg in vulnerability.fixed_by_packages %} - {% if pkg in test_get_fixing_purls %} -
- {{ pkg.purl }} -
- {% endif %} - {% endfor %} - {% empty %} - This package is not known to be affected by vulnerabilities. diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index eb181ff93..6a41620b3 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -83,40 +83,7 @@ def get_context_data(self, **kwargs): context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) - context["test01"] = [ - "aaa", - "bbb", - "ccc", - ] - context["test02"] = [ - package, - package.type, - package.namespace, - package.name, - package.version, - package.qualifiers, - package.subpath, - ] - context["test03"] = package.affected_by - - # ======================================================== - context["test04"] = [ - fixing_pkg - for vuln in package.affected_by - # for fixing_pkg in vuln.test_matching_fixed_by_packages - for fixing_pkg in vuln.fixed_by_packages - if fixing_pkg.type == package.type - and fixing_pkg.namespace == package.namespace - and fixing_pkg.name == package.name - and fixing_pkg.qualifiers == package.qualifiers - and fixing_pkg.subpath == package.subpath - # I think this version comparison requires the use of univers. - # Plus we just need the closest one, not all that are greater than. - and fixing_pkg.version > package.version - ] - # ======================================================== - context["test_get_fixing_purls"] = package.test_get_fixing_purls - # ======================================================== + context["get_fixing_packages"] = package.get_fixing_packages return context From 6497e90014ab4a5f8370005b827929a38cb180ed Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 7 Aug 2023 19:04:39 -0700 Subject: [PATCH 04/53] Update affected-fixed package matching #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 663 ++++++++++++++++-- .../templates/package_details.html | 140 +++- vulnerabilities/tests/test_models.py | 65 ++ vulnerabilities/views.py | 5 +- vulnerablecode/settings.py | 4 +- vulnerablecode/static/css/custom.css | 102 ++- 6 files changed, 929 insertions(+), 50 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index dde000813..d07a0530f 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -635,100 +635,685 @@ def get_fixed_packages(self, package): packagerelatedvulnerability__fix=True, ).distinct() - def assign_univers_version(self, fixing_pkg): + def get_sibling_packages(self, package): """ - Identify which univers version applies to the two packages to be compared (self and a fixing package), - evaluate whether the fixing_pkg version is > than the target affected package, and + Return a queryset of all packages with the same type, namespace, name, subpath and qualifiers of the `package`, whether or not they fix any vulnerability + """ + return Package.objects.filter( + name=package.name, + namespace=package.namespace, + type=package.type, + qualifiers=package.qualifiers, + subpath=package.subpath, + # packagerelatedvulnerability__fix=True, + ).distinct() + + # def assign_and_compare_univers_versions(self, fixed_pkg): + def assign_univers_version(self, fixed_pkg): + """ + Identify which univers version applies to the two packages to be compared (self and a fixed package), + evaluate whether the fixed_pkg version is > than the target affected package, and return True or False. """ - # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. + # TODO: Instead of return True or False based on evaluating the incoming fixed_pkg type to a univers version and then checking whether the fixed version is greater than the affected (self) version, we'll just use the incoming type to assign and return a univers version -- as command_name -- to be used in get_closest_fixed_package() below to add all greater-than versions to the later_matching_fixed_packages list -- which in turn will be fed to self.sort_by_version(later_matching_fixed_packages) # Many more to be added. match_type_to_univers_version = { "conan": versions.ConanVersion, + "deb": versions.DebianVersion, "maven": versions.MavenVersion, + "openssl": versions.OpensslVersion, + "pypi": versions.PypiVersion, } command_name = "" - type_to_version = match_type_to_univers_version.get(fixing_pkg.type) - if type_to_version: + matched_type_to_version = match_type_to_univers_version.get(fixed_pkg.type) + if matched_type_to_version: print("\t--------------------------") - print("\ttype_to_version = {}\n".format(type_to_version)) - command_name = type_to_version + print("*** matched_type_to_version = {}".format(matched_type_to_version)) + command_name = matched_type_to_version else: - print("\ttype_to_version = NO MATCH\n") - command_name = versions.Version + print("\t--------------------------") + print("*** matched_type_to_version = NO MATCH") + # Using "command_name = versions.Version", the test + # assert versions.Version("0.9") < versions.Version("0.10") + # fails! + # command_name = versions.Version + # Use this as a default fallback instead. + command_name = versions.SemverVersion + + # if command_name(fixed_pkg.version) > command_name(self.version): + # return True + # else: + # return False + + # Instead return command_name for recipient to use as needed for sorting or perhaps other uses + return command_name + + def sort_by_version(self, later_matching_fixed_packages): + # Incoming is a list of + + # ALERT: added this to address server error 500 but related error arose: line 908, in get_closest_fixed_package + # HOT: How is this related to the source of the server error (500)? + if len(later_matching_fixed_packages) == 0: + return + + # Replace find_closest_fixed_by_package()? + print("\nlater_matching_fixed_packages = {}".format(later_matching_fixed_packages)) + print("\nlater_matching_fixed_packages[0] = {}".format(later_matching_fixed_packages[0])) + print( + "\ntype(later_matching_fixed_packages[0]) = {}".format( + type(later_matching_fixed_packages[0]) + ) + ) + # NOTE: This gives us the PURL type but instead we want the PURL itself to pass to assign_univers_version(self, fixed_pkg) and get the command_name in return, which we'll then use in the sort process. + print( + "\nlater_matching_fixed_packages[0].type = {}".format( + later_matching_fixed_packages[0].type + ) + ) + print( + "\ntype(later_matching_fixed_packages[0].type) = {}".format( + type(later_matching_fixed_packages[0].type) + ) + ) - if command_name(fixing_pkg.version) > command_name(self.version): - return True - else: - return False + # Incoming is a list -- later_matching_fixed_packages + + # We'll use assign_univers_version() above to get the univers version as a command_name. + # But what do we pass to it? The [0] index of the incoming list, i.e., later_matching_fixed_packages[0]? + command_name = self.assign_univers_version(later_matching_fixed_packages[0]) + + print("\n>>> command_name = {}\n".format(command_name)) + + # TODO: Maybe we don't need to convert to a PURL, a list of dictionaries etc.?? + print( + "\n+++++++ later_matching_fixed_packages[0].version = {}".format( + later_matching_fixed_packages[0].version + ) + ) + + # sort + test_sort_by_version = [] + test_sort_by_version = sorted( + # later_matching_fixed_packages, key=lambda x: versions.DebianVersion(x["version"]) + later_matching_fixed_packages, + # key=lambda x: versions.MavenVersion(x.version), + key=lambda x: command_name(x.version), + ) + + print("\ntest_sort_by_version = {}\n".format(test_sort_by_version)) + + return test_sort_by_version + + # convert_to_dict_list = [] + + # sorted_later_matching_fixed_packages = [] + + # # TODO: First, convert to a list of dictionaries. + # for pkg in later_matching_fixed_packages: + # # pkg is a + # print("pkg = {}".format(pkg)) + # print("type(pkg) = {}".format(type(pkg))) + + # # pkg_str is a string + # pkg_str = pkg.package_url + # print("pkg_str = {}".format(pkg_str)) + # print("type(pkg_str) = {}".format(type(pkg_str))) + + # # purl is a + # purl = PackageURL.from_string(pkg_str) + # print("purl = {}".format(purl)) + # print("type(purl) = {}".format(type(purl))) + + # purl_dict = purl.to_dict() + # print("purl_dict = {}".format(purl_dict)) + # print("type(purl_dict) = {}".format(type(purl_dict))) + + # convert_to_dict_list.append(purl.to_dict()) + # print("HELLO\n") + + # print("\nconvert_to_dict_list = {}\n".format(convert_to_dict_list)) + + # return convert_to_dict_list + # ========================================================== + # sorted_later_matching_fixed_packages = sorted( + # later_matching_fixed_packages, key=lambda x: versions.MavenVersion(x["version"]) + # ) + # print( + # "\nsorted_later_matching_fixed_packages = {}\n".format( + # sorted_later_matching_fixed_packages + # ) + # ) + # print("\n".join(map(str, sorted_later_matching_fixed_packages))) + + # return what? + + # ========================================================== + # ========================================================== + + # def find_closest_fixed_by_package(self, later_matching_fixed_packages): + # # Maybe use sort_by_version() above instead? + # # take the incoming list later_matching_fixed_packages, convert to list of dictionaries, sort by version using univers.version.[version class], choose the top i.e., index [0] and convert back to PURL and return that PURL. + # print("\nlater_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) + + # closest_fixed_by_package = "TBD" + + # return closest_fixed_by_package @property - def get_fixing_packages(self): + # def get_fixing_packages(self): + def get_closest_fixed_package(self): """ - This function identifies the closest fixing version that is greater than the affected version and + This function identifies the closest fixed package version that is greater than the affected package version and is the same type, namespace, name, qualifiers and subpath as the affected package. """ print("\nself = {}\n".format(self)) - # This returns all fixing packages that match the target package (type etc.), regardless of fixed vuln. - fixed_packages = self.get_fixed_packages(package=self) + # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. + # fixed_packages = self.get_fixed_packages(package=self) + # This is clearer. + matching_fixed_packages = self.get_fixed_packages(package=self) # This returns a list of the vulnerabilities that affect this package (i.e., self). qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - # This takes the list of vulns affecting the current package, retrieves a list of the fixing packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages`. - # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). + # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). qs = qs.prefetch_related( Prefetch( "packages", - queryset=fixed_packages, - to_attr="filtered_fixed_packages", + # queryset=fixed_packages, + queryset=matching_fixed_packages, + # to_attr="filtered_fixed_packages", + to_attr="matching_fixed_packages", ) ) # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). print("qs = {}\n".format(qs)) - prefetch_fixed_packages = [] + # ************************************************************************ + + later_matching_fixed_packages = [] vuln_count = 0 for vuln in qs: print("vuln = {}\n".format(vuln)) + # print( + # "\tqs[vuln_count].filtered_fixed_packages = {}".format( + # qs[vuln_count].filtered_fixed_packages + # ) + # ) print( - "\tqs[vuln_count].filtered_fixed_packages = {}".format( - qs[vuln_count].filtered_fixed_packages + "\tqs[vuln_count].matching_fixed_packages = {}".format( + qs[vuln_count].matching_fixed_packages ) ) print("") # Check the Prefetch qs. - # TODO: Do we want to check whether the fixing version has any vulnerabilities of its own? - for fixing_pkg in qs[vuln_count].filtered_fixed_packages: - print("\tfixing_pkg = {}".format(fixing_pkg)) - print("\tfixing_pkg.type = {}".format(fixing_pkg.type)) - print("\tfixing_pkg.version = {}".format(fixing_pkg.version)) + # TODO: Do we want to check whether the fixed version has any vulnerabilities of its own? + # for fixed_pkg in qs[vuln_count].filtered_fixed_packages: + for fixed_pkg in qs[vuln_count].matching_fixed_packages: + print("\tfixed_pkg = {}".format(fixed_pkg)) + print("\tfixed_pkg.type = {}".format(fixed_pkg.type)) + print("\tfixed_pkg.version = {}".format(fixed_pkg.version)) print("\t--------------------------") print("\tself.type = {}".format(self.type)) print("\tself.version = {}".format(self.version)) - # Assign univers version and compare: False = fixing_pkg.version < self.version (affected version). - # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. - immediate_fix = self.assign_univers_version(fixing_pkg) - print("\t--------------------------") - print("\timmediate_fix = {}\n".format(immediate_fix)) - - if fixing_pkg in fixed_packages and immediate_fix: - prefetch_fixed_packages.append(fixing_pkg) + # Assign univers version and compare: False = fixed_pkg.version < self.version (affected version). + + # 2023-08-02 Wednesday 16:01:35. atm immediate_fix is True or False. If instead assign_and_compare_univers_versions() returns the univers version, we could get that here and then test with this or similar right here -- enabling use of the univers version function in other places as well, like a sort_by_version function! + # if command_name(fixed_pkg.version) > command_name(self.version): + # return True + # else: + # return False + # ===================================================== + # Replace this with chunk below + # immediate_fix = self.assign_and_compare_univers_versions(fixed_pkg) + # print("\t--------------------------") + # print("\timmediate_fix = {}\n".format(immediate_fix)) + + # if fixed_pkg in fixed_packages and immediate_fix: + # later_matching_fixed_packages.append(fixed_pkg) + # ===================================================== + # command_name = self.assign_and_compare_univers_versions(fixed_pkg) + # renamed + # TODO: Move this up before the for loop -- both for loops if possible -- to reduce calls! + command_name = self.assign_univers_version(fixed_pkg) + print("\nJust requested command_name >>> {}\n".format(command_name)) + # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( + # self.version + # ): + if fixed_pkg in matching_fixed_packages and command_name( + fixed_pkg.version + ) > command_name(self.version): + later_matching_fixed_packages.append(fixed_pkg) vuln_count += 1 - return prefetch_fixed_packages + # find_closest_fixed_by_package -- from the list later_matching_fixed_packages + # closest_fixed_by_package = self.find_closest_fixed_by_package(later_matching_fixed_packages) + + # TODO: or instead use this. This will be a list sorted by univers version class, and here all we need is to grab the [0] index from that list for the closest fixed by package! So we'd return a single closest_fixed_package. + # ALERT: The sort query needs to be done separately for each vulnerability because the list of fixed by packages is likely to be different. As is, we return a single sorted list of all fixed by packages for the affected package and then pass just the [0] package -- not what we want to do! + sort_fixed_by_packages_by_version = self.sort_by_version(later_matching_fixed_packages) + print( + "\nsort_fixed_by_packages_by_version = {}\n".format(sort_fixed_by_packages_by_version) + ) + # ALERT: 2023-08-05 Saturday 23:22:08. Address server error 500? + # ALERT: 2023-08-05 Saturday 23:24:50. This actusally fixed the server error (500) and I can now even see the Packafe details page for pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1 !!! + # HOT: I need to trace back the root cause of the server error (500). I suspect it's something like a record with no fixed by packages or something else that is an empty list but which i try to measure, e.g., for a print statement, or possibly for a real if condition.. + if sort_fixed_by_packages_by_version is None: + return + + closest_fixed_package = sort_fixed_by_packages_by_version[0] + + # print("\n!!! later_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) + # print( + # "\n!!! sort_fixed_by_packages_by_version = {}\n".format( + # sort_fixed_by_packages_by_version + # ) + # ) + # print("\n!!! closest_fixed_package = {}\n".format(closest_fixed_package)) + + # rebuilt_purl_from_dict = PackageURL( + # closest_fixed_package["type"], + # closest_fixed_package["namespace"], + # closest_fixed_package["name"], + # closest_fixed_package["version"], + # closest_fixed_package["qualifiers"], + # closest_fixed_package["subpath"], + # ) + # print("\n!!! rebuilt_purl_from_dict = {}\n".format(rebuilt_purl_from_dict)) + + # return later_matching_fixed_packages + return sort_fixed_by_packages_by_version + # return [closest_fixed_package] + + # return [rebuilt_purl_from_dict] + + @property + def fixed_package_details(self): + """ + This is a test that might develop into a model-based equivalent of the loops etc. I was doing/trying to do in the Jinja2 template. I'm going to add this as a context so we can see it in the template. + """ + # return "Hello" + + # vcio_dict = { + # [ + # {"VCID-2nyb-8rwu-aaag": "PURL01"}, + # {"VCID-gqhw-ngh8-aaap": "PURL02"}, + # {"some-other-id": "PURL03"}, + # ] + # } + + print("\n==> This is from the test_property_01() property.\n") + + print("\nself = {}\n".format(self)) + + # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. + # fixed_packages = self.get_fixed_packages(package=self) + # This is clearer. + matching_fixed_packages = self.get_fixed_packages(package=self) + + # This returns a list of the vulnerabilities that affect this package (i.e., self). + qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) + + # TODO: Can we get all sibling packages so that we can then determine which have 0 vulnerabilities and of these the closest and maybe the most recent as well? + + all_sibling_packages = self.get_sibling_packages(package=self) + print("\nall_sibling_packages = {}\n".format(all_sibling_packages)) + print("\nlen(all_sibling_packages) = {}\n".format(len(all_sibling_packages))) + + non_vuln_sibs = [] + for sib in all_sibling_packages: + if sib.is_vulnerable is False: + non_vuln_sibs.append(sib) + print("\nnon_vuln_sibs = {}\n".format(non_vuln_sibs)) + print("\nlen(non_vuln_sibs) = {}\n".format(len(non_vuln_sibs))) + + # Add just the greater-than versions to a new list + command_name = self.assign_univers_version(self) + print( + "\nOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO command_name = {}\n".format( + command_name + ) + ) + later_non_vuln_sibs = [] + for non_vuln_sib in non_vuln_sibs: + if command_name(non_vuln_sib.version) > command_name(self.version): + later_non_vuln_sibs.append(non_vuln_sib) + + print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) + print("\nlen(later_non_vuln_sibs) = {}\n".format(len(later_non_vuln_sibs))) + + # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). + # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + qs = qs.prefetch_related( + Prefetch( + "packages", + # queryset=fixed_packages, + queryset=matching_fixed_packages, + # to_attr="filtered_fixed_packages", + to_attr="matching_fixed_packages", + ) + ) + + # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). + print("\nzzz qs = {}\n".format(qs)) + + purl_dict = {} + + purl_dict["purl"] = self.purl + + purl_dict.update({"vulnerabilities": []}) + + # purl_dict["vulnerabilities"].append({"fruit": "orange"}) + + for vuln in qs: + print("\nzzz vuln = {}\n".format(vuln)) + print("\nzzz type(vuln) = {}\n".format(type(vuln))) + + later_matching_fixed_packages = [] + # purl_dict[vuln.vulnerability_id] = "aaa" + # purl_dict.update({"vulnerability": vuln.vulnerability_id}) + + purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + + # TODO:2023-08-05 Saturday 13:12:28. This returns a list of matching fixed packages for this specific vuln! + vuln_matching_fixed_packages = vuln.matching_fixed_packages + print("\nzzz self.purl = {}\n".format(self.purl)) + print("\nzzz vuln = {}\n".format(vuln)) + print("\nzzz vuln_matching_fixed_packages = {}\n".format(vuln_matching_fixed_packages)) + + # TODO: So we need to sort this list by version using the correct univers version and then return the [0] index in that sorted list + # QUESTION: Do we still need to remove lesser-than fixed packages or did we already do that? + + # ============================================================= + # command_name = self.assign_univers_version(fixed_pkg) + command_name = self.assign_univers_version(self) + print("\nzzz command_name = {}\n".format(command_name)) + + # ALERT: What if there are no fixed by packages? The following thows an error because the list 'vuln_matching_fixed_packages' is empty! + # [I fixed this, right? ;-] + + closest_fixed_package = "" + + if len(vuln_matching_fixed_packages) > 0: + + for fixed_pkg in vuln_matching_fixed_packages: + if fixed_pkg in matching_fixed_packages and command_name( + fixed_pkg.version + ) > command_name(self.version): + later_matching_fixed_packages.append(fixed_pkg) + + # print("\nJust requested command_name >>> {}\n".format(command_name)) + # # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( + # # self.version + # # ): + # if fixed_pkg in matching_fixed_packages and command_name( + # fixed_pkg.version + # ) > command_name(self.version): + # later_matching_fixed_packages.append(fixed_pkg) + # ============================================================= + # later_matching_fixed_packages = vuln.matching_fixed_packages + + print( + "\nzzz later_matching_fixed_packages = {}\n".format( + later_matching_fixed_packages + ) + ) + + sort_fixed_by_packages_by_version = self.sort_by_version( + later_matching_fixed_packages + ) + print( + "\nzzz sort_fixed_by_packages_by_version = {}\n".format( + sort_fixed_by_packages_by_version + ) + ) + closest_fixed_package = sort_fixed_by_packages_by_version[0] + # closest_fixed_package = sort_fixed_by_packages_by_version[0].purl + # 2023-08-06 Sunday 11:15:03. This returns a queryset of vulns affecting this package. + # HOT: How do we get the closest fixed by package vuln count and list of vulns? I keep getting errors. ;-) + closest_fixed_package_vulns = closest_fixed_package.affected_by + # closest_fixed_package_vulns_list = list(closest_fixed_package_vulns) + # ALERT: 2023-08-06 Sunday 14:25:49. This did the trick! + + # closest_fixed_package_vulns_list = [ + # i.vulnerability_id for i in closest_fixed_package_vulns + # ] + + # 2023-08-06 Sunday 16:53:27. Try a named tuple to pass the vuln's vulnerability+id and get_absolute_url. + # FixedPackageVuln = namedtuple("FixedPackageVuln", "vuln_id, vuln_get_absolute_url") + # closest_fixed_package_vulns_list = [ + # FixedPackageVuln( + # vuln_id=fixed_pkg_vuln.vulnerability_id, + # vuln_get_absolute_url=fixed_pkg_vuln.get_absolute_url(), + # ) + # for fixed_pkg_vuln in closest_fixed_package_vulns + # ] + # ALERT: Replace the namedtuple with a dict -- this way it can be added to the purl_dict as a nested dict rather than a list of 2 values. + closest_fixed_package_vulns_dict = [ + { + "vuln_id": fixed_pkg_vuln.vulnerability_id, + "vuln_get_absolute_url": fixed_pkg_vuln.get_absolute_url(), + } + for fixed_pkg_vuln in closest_fixed_package_vulns + ] + + # === + # closest_fixed_package_vulns_list = closest_fixed_package_vulns.objects.values_list() + + # closest_fixed_package_vulns_list = [] + # for closest_vuln in closest_fixed_package_vulns: + # closest_fixed_package_vulns_list.append(closest_vuln) + # print( + # "\t\nQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ vuln = {}\n".format( + # closest_vuln + # ) + # ) + # print("\t\ntype(closest_vuln) = {}".format(type(closest_vuln))) + + # # closest_fixed_package_vuln_count = len(closest_fixed_package.affected_by) + # print( + # "\t\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type(closest_fixed_package_vulns) = {}\n".format( + # type(closest_fixed_package_vulns) + # ) + # ) + + else: + closest_fixed_package = "There are no reported fixed packages." + # Is None the value we want? We do not want to display anything but the count = 0. + # closest_fixed_package_vulns = None + # closest_fixed_package_vuln_count = 0 + + # closest_fixed_package_vulns_list = [] + + print("\nzzz closest_fixed_package = {}".format(closest_fixed_package)) + print("zzz type(closest_fixed_package) = {}\n".format(type(closest_fixed_package))) + + # TODO: How do we add 'closest_fixed_by_purl', 'closest_fixed_by_vulns' and 'non_vulnerable_fix'? + + # # for vuln in purl_dict["vulnerabilities"]: + # # # vuln["closest_fixed_by_purl"] = "?????" + # # vuln["closest_fixed_by_purl"] = closest_fixed_package + # # vuln["closest_fixed_by_url"] = "?????" + # # vuln["closest_fixed_by_vulnerabilities"] = "?????" + # # vuln["non_vulnerable_fix"] = "?????" + # # vuln["non_vulnerable_fix_url"] = "?????" + + for dict_vuln in purl_dict["vulnerabilities"]: + print("\n===================================> vuln = {}\n".format(vuln)) + print("\n===================================> type(vuln) = {}\n".format(type(vuln))) + print("\n===================================> vuln.vcid = {}\n".format(vuln.vcid)) + print( + "\n===================================> dict_vuln['vulnerability'] = {}\n".format( + dict_vuln["vulnerability"] + ) + ) + # TODO: Up above we defined 'non_vuln_sibs' but we still need to remove those with less than version + # ALERT: remove less than versions from 'non_vuln_sibs' + # 2023-08-05 Saturday 20:30:47. Hopefully just wrote the code for that up above, with the new list 'later_non_vuln_sibs'. + + closest_non_vulnerable_fix = "" + # if len(non_vuln_sibs) > 0: + # closest_non_vulnerable_fix = self.sort_by_version(non_vuln_sibs)[0] + if len(later_non_vuln_sibs) > 0: + closest_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[0] + # else: + # # closest_non_vulnerable_fix = ( + # # "There are no reported non-vulnerable fixed packages." + # # ) + # closest_non_vulnerable_fix = None + + most_recent_non_vulnerable_fix = "" + if len(later_non_vuln_sibs) > 0: + most_recent_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[-1] + else: + # most_recent_non_vulnerable_fix = ( + # "There are no reported non-vulnerable fixed packages." + # ) + most_recent_non_vulnerable_fix = None + + # if dict_vuln["vulnerability"] == vuln.vulnerability_id: + if dict_vuln["vulnerability"] == str(vuln): + # if dict_vuln["vulnerability"] == vuln.vcid: + # dict_vuln["closest_fixed_by_purl"] = "?????" + dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) + dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() + # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vuln_count + # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns + # dict_vuln["closest_fixed_by_vulnerabilities"] = ["A", "B"] + + # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_list + # ALERT: Replace the above list created with a namedtuple with the following dictionary: + dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_dict + # ALERT: Moved these up 1 level in the dict. + # dict_vuln["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + # dict_vuln[ + # "closest_non_vulnerable_fix_url" + # ] = closest_non_vulnerable_fix.get_absolute_url() + # dict_vuln["most_recent_non_vulnerable_fix"] = str( + # most_recent_non_vulnerable_fix + # ) + # dict_vuln[ + # "most_recent_non_vulnerable_fix_url" + # ] = most_recent_non_vulnerable_fix.get_absolute_url() + + # QUESTION: Can we add the non-vuln data as higher-level key-value pairs rather than children of "vulnerabilities"? + + # purl_dict.update({"fruits": []}) + # purl_dict["fruits"].append({"fruit": "apple"}) + # purl_dict["fruits"].append({"fruit": "banana"}) + + # purl_dict.update( + # {"closest_non_vulnerable_fix": str(closest_non_vulnerable_fix)} + # ) + # purl_dict.update( + # { + # "closest_non_vulnerable_fix_url": closest_non_vulnerable_fix.get_absolute_url() + # } + # ) + # purl_dict.update( + # {"most_recent_non_vulnerable_fix": str(most_recent_non_vulnerable_fix)} + # ) + # purl_dict.update( + # { + # "most_recent_non_vulnerable_fix_url": most_recent_non_vulnerable_fix.get_absolute_url() + # } + # ) + + purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + purl_dict[ + "closest_non_vulnerable_fix_url" + ] = closest_non_vulnerable_fix.get_absolute_url() + purl_dict["most_recent_non_vulnerable_fix"] = str( + most_recent_non_vulnerable_fix + ) + purl_dict[ + "most_recent_non_vulnerable_fix_url" + ] = most_recent_non_vulnerable_fix.get_absolute_url() + + print("\npurl_dict = {}\n".format(purl_dict)) + + print(json.dumps(purl_dict, indent=4, sort_keys=False)) + + # # Print to text file + pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) + # logger = logging.getLogger(__name__) + # logger.setLevel(logging.INFO) + # # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt")) + # # will this overwrite prior writes? weird output + # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt", mode="w")) + # logger.info(pretty_purl_dict) + + with open("/home/jmh/pretty_purl_dict.txt", "w") as f: + f.write(pretty_purl_dict) + + alternate_dict_01 = { + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "vulnerabilities": [ + { + "vulnerability": "VCID-2nyb-8rwu-aaag", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + # "get_absolute_url": reverse("package_details", args=[self.purl]), + "closest_fixed_by_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2"], + ), + "closest_fixed_by_vulns": 2, + "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "non_vulnerable_fix_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + }, + { + "vulnerability": "VCID-gqhw-ngh8-aaap", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4", + # "get_absolute_url": reverse("package_details", args=[self.purl]), + "closest_fixed_by_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"], + ), + "closest_fixed_by_vulns": 1, + "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "non_vulnerable_fix_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + }, + { + "vulnerability": "VCID-t7e4-g3fr-aaan", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + # "get_absolute_url": reverse("package_details", args=[self.purl]), + "closest_fixed_by_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + "closest_fixed_by_vulns": 0, + "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "non_vulnerable_fix_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + }, + ], + } + + # return vcio_dict + + # return alternate_dict_01 + + return purl_dict class PackageRelatedVulnerability(models.Model): diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index cfbfdd706..8d204bccf 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -41,7 +41,6 @@
-
During testing, red = all packages that fix the vulnerability (for pkg in vulnerability.fixed_by_packages) and green = the closest fixing package whose version is greater than the affected package's version (if pkg in get_fixing_packages).
Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }})
@@ -52,7 +51,7 @@ Vulnerability Summary Aliases - Fixing Packages + Fixed by packages @@ -76,22 +75,147 @@ {% endif %} {% endfor %} - + + + + + + + + {% if package.purl in fixed_package_details.purl %} + {% for key, value in fixed_package_details.items %} + {% if key == "vulnerabilities" %} + {% for abc in value %} + {% if abc.vulnerability == vulnerability.vulnerability_id %} +
    +
  • + PURL: {{ fixed_package_details.purl }} + + + +
    • +
  • + Vulnerability: {{ abc.vulnerability }} +
    • +
  • + Closest fixed-by PURL: + {{ abc.closest_fixed_by_purl }} +
    • +
  • + Closest fixed-by vulnerability count: {{ abc.closest_fixed_by_vulnerabilities|length }} + {% if abc.closest_fixed_by_vulnerabilities|length != 0 %} +
    +
    + +
    + +
    + {% endif %} +
    • +
  • + Closest non-vulnerable fix: + + + {{ fixed_package_details.closest_non_vulnerable_fix }} +
    • +
  • + Most recent non-vulnerable fix: + + + {{ fixed_package_details.most_recent_non_vulnerable_fix }} +
    • +
+ {% endif %} + {% endfor %} + + + {% endif %} + {% endfor %} + {% else %} + NO-- {{ package.purl }} + {% endif %} + + + + + +
@@ -109,7 +233,7 @@
- Fixing vulnerabilities ({{ fixing_vulnerabilities|length }}) + Fixed by vulnerabilities ({{ fixing_vulnerabilities|length }})
diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 58b95af80..24a66d2d2 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -7,12 +7,14 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import urllib.parse from datetime import datetime from unittest import TestCase import pytest from django.db.utils import IntegrityError from freezegun import freeze_time +from univers import versions from vulnerabilities import models @@ -88,3 +90,66 @@ def test_vulnerability_package(self): assert v1.vulnerable_packages.all()[0] == p1 assert v1.patched_packages.all()[0] == p2 + + +@pytest.mark.django_db +class TestPackageModel(TestCase): + def test_univers_version_comparisons(self): + assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") + + assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") + + # pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 is a real PURL in the DB + # But I get an error when I try to compare 2 PURLs with the same suffix -- + # univers.versions.InvalidVersion: '2.12.1-1%2Bdeb11u1' is not a valid + # Do we need to replace/delete the "%"? + # assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( + # "2.13.1-1%2Bdeb11u1" + # ) + # Test the error + with pytest.raises(versions.InvalidVersion): + assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( + "2.13.1-1%2Bdeb11u1" + ) + # Decode the version and test. + assert versions.DebianVersion( + urllib.parse.unquote("2.12.1-1%2Bdeb11u1") + ) < versions.DebianVersion(urllib.parse.unquote("2.13.1-1%2Bdeb11u1")) + + with pytest.raises(TypeError): + assert versions.PypiVersion("0.9") < versions.DebianVersion("0.10") + + # Using versions.Version does not correctly make this comparison! + assert not versions.Version("0.9") < versions.Version("0.10") + # Use SemverVersion instead as a default fallback version for comparisons. + assert versions.SemverVersion("0.9") < versions.SemverVersion("0.10") + + def test_assign_and_compare_univers_versions(self): + deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") + deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") + + immediate_fix01 = deb01.assign_and_compare_univers_versions(deb02) + print("\nimmediate_fix01 = {}\n".format(immediate_fix01)) + # assert deb01.assign_and_compare_univers_versions(deb02) is True + assert deb01.assign_and_compare_univers_versions(deb02) + + immediate_fix02 = deb02.assign_and_compare_univers_versions(deb01) + print("\nimmediate_fix02 = {}\n".format(immediate_fix02)) + # assert deb02.assign_and_compare_univers_versions(deb01) is False + assert not deb02.assign_and_compare_univers_versions(deb01) + + pypi01 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") + pypi02 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.10") + + immediate_fix03 = pypi01.assign_and_compare_univers_versions(pypi02) + print("\nimmediate_fix03 = {}\n".format(immediate_fix03)) + # assert pypi01.assign_and_compare_univers_versions(pypi02) is True + assert pypi01.assign_and_compare_univers_versions(pypi02) + + gem01 = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") + gem02 = models.Package.objects.create(type="gem", name="sidekiq", version="0.10") + + immediate_fix04 = gem01.assign_and_compare_univers_versions(gem02) + print("\nimmediate_fix04 = {}\n".format(immediate_fix04)) + # assert gem01.assign_and_compare_univers_versions(gem02) is True + assert gem01.assign_and_compare_univers_versions(gem02) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 6a41620b3..396ac1bc2 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -83,7 +83,10 @@ def get_context_data(self, **kwargs): context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) - context["get_fixing_packages"] = package.get_fixing_packages + # context["get_fixing_packages"] = package.get_fixing_packages + context["get_closest_fixed_package"] = package.get_closest_fixed_package + # context["test_property_01"] = package.test_property_01 + context["fixed_package_details"] = package.fixed_package_details return context diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 3187b67ec..0071686fb 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -39,7 +39,9 @@ CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS", default=[]) # SECURITY WARNING: do not run with debug turned on in production -DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) +# DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) +# DEBUG = "127.0.0.1" +DEBUG = True # SECURITY WARNING: do not run with debug turned on in production DEBUG_TOOLBAR = env.bool("VULNERABLECODE_DEBUG_TOOLBAR", default=False) diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index 6699d8977..96ef5aad3 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -329,6 +329,14 @@ a.small_page_button { box-shadow: 0px 8px 16px 0px #808080; } +.dropdown-vuln-list-width { + width: 400px; +} + +.dropdown-vuln-dict-width { + max-width: 600px; +} + .width-100-pct { width: 100%; } @@ -368,4 +376,96 @@ a.small_page_button { span.tag.custom { margin: 0px 0px 6px 10px; -} \ No newline at end of file +} + +/* test bulleted list */ + +ul.fixed_by_bullet { + list-style-type: disc; + /*margin-top: 2px; +margin-bottom: 10px;*/ + /*margin-left: -24px;*/ + /*margin-left: -30px;*/ + margin-top: 0.25em; + margin-left: 7px; + margin-bottom: 0.25em; + padding-left: 10px; +} + +ul.fixed_by_bullet ul { + list-style-type: disc; + /*margin-top: 10px;*/ + margin-top: 5px; + margin-top: 0px; + margin-bottom: 0px; + margin-left: 23px; + margin-left: 18px; + padding: 0; + border: none; +} + + + +ul.fixed_by_bullet li { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + /*margin-bottom: 10px;*/ + margin-bottom: 2px; +} + +ul.fixed_by_bullet li:last-child { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + /*margin-bottom: 10px;*/ + margin-bottom: 0px; +} + +ul.fixed_by_bullet li li { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + margin-top: 0px; + color: #000000; +} + +/* 10/10/15 add 3rd-level bullets */ +ul.fixed_by_bullet ul ul { + list-style-type: disc; + margin-top: 0px; + margin-bottom: 0px; + margin-left: 50px; + margin-left: 17px; + padding: 0; + border: none; +} + +ul.fixed_by_bullet li li li { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + margin-top: 0px; + color: #000000; +} + +/* CSS for dev fixed by headers */ +.dev_fixed_by_headers { + border: solid 1px #cccccc; + border-radius: 3px; + background-color: #f2f2f2; + color: #000000; + font-weight: bold; + font-size: 13px; + padding: 3px; + margin-bottom: 3px; + display: block; +} From b6dba7826706c544bce36dbb4c642d1d5cc46416 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Wed, 9 Aug 2023 18:47:51 -0700 Subject: [PATCH 05/53] Improve matching and reporting code and UI #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 583 ++---------------- .../templates/package_details.html | 210 +++---- vulnerabilities/tests/test_models.py | 45 +- vulnerabilities/views.py | 3 - vulnerablecode/settings.py | 4 +- vulnerablecode/static/css/custom.css | 109 +--- 6 files changed, 154 insertions(+), 800 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index d07a0530f..b8b2bea94 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -637,7 +637,8 @@ def get_fixed_packages(self, package): def get_sibling_packages(self, package): """ - Return a queryset of all packages with the same type, namespace, name, subpath and qualifiers of the `package`, whether or not they fix any vulnerability + Return a queryset of all packages with the same type, namespace, name, subpath and + qualifiers as the target package, whether or not the 'sibling packages' fix any vulnerability. """ return Package.objects.filter( name=package.name, @@ -645,462 +646,126 @@ def get_sibling_packages(self, package): type=package.type, qualifiers=package.qualifiers, subpath=package.subpath, - # packagerelatedvulnerability__fix=True, ).distinct() - # def assign_and_compare_univers_versions(self, fixed_pkg): def assign_univers_version(self, fixed_pkg): """ - Identify which univers version applies to the two packages to be compared (self and a fixed package), - evaluate whether the fixed_pkg version is > than the target affected package, and - return True or False. + Identify which univers version applies to a package and return that version for use, e.g., + in sorting a group of sibling packages (same type etc.). """ - - # TODO: Instead of return True or False based on evaluating the incoming fixed_pkg type to a univers version and then checking whether the fixed version is greater than the affected (self) version, we'll just use the incoming type to assign and return a univers version -- as command_name -- to be used in get_closest_fixed_package() below to add all greater-than versions to the later_matching_fixed_packages list -- which in turn will be fed to self.sort_by_version(later_matching_fixed_packages) - - # Many more to be added. + # TODO: Many more to be added. match_type_to_univers_version = { "conan": versions.ConanVersion, "deb": versions.DebianVersion, + # The following throws an error: AttributeError: module 'univers.versions' has no attribute 'GemVersion' + # "gem": versions.GemVersion, "maven": versions.MavenVersion, + "nginx": versions.NginxVersion, + # "npm": "openssl": versions.OpensslVersion, "pypi": versions.PypiVersion, + # from https://github.com/nexB/univers/blob/205da7ecbf7b0f195662373ea710b2b84a877eb0/tests/test_version_comparison.py + # versions.SemverVersion, + # versions.GolangVersion, + # versions.PypiVersion, + # versions.GenericVersion, + # versions.ComposerVersion, + # versions.NginxVersion, + # versions.ArchLinuxVersion, + # versions.DebianVersion, + # versions.RpmVersion, + # versions.MavenVersion, + # versions.NugetVersion, + # versions.GentooVersion, + # versions.OpensslVersion, + # versions.LegacyOpensslVersion, + # versions.AlpineLinuxVersion, } command_name = "" - matched_type_to_version = match_type_to_univers_version.get(fixed_pkg.type) if matched_type_to_version: - print("\t--------------------------") - print("*** matched_type_to_version = {}".format(matched_type_to_version)) command_name = matched_type_to_version - else: - print("\t--------------------------") - print("*** matched_type_to_version = NO MATCH") - # Using "command_name = versions.Version", the test - # assert versions.Version("0.9") < versions.Version("0.10") - # fails! - # command_name = versions.Version - # Use this as a default fallback instead. command_name = versions.SemverVersion - # if command_name(fixed_pkg.version) > command_name(self.version): - # return True - # else: - # return False - - # Instead return command_name for recipient to use as needed for sorting or perhaps other uses return command_name def sort_by_version(self, later_matching_fixed_packages): # Incoming is a list of - - # ALERT: added this to address server error 500 but related error arose: line 908, in get_closest_fixed_package - # HOT: How is this related to the source of the server error (500)? - if len(later_matching_fixed_packages) == 0: - return - - # Replace find_closest_fixed_by_package()? - print("\nlater_matching_fixed_packages = {}".format(later_matching_fixed_packages)) - print("\nlater_matching_fixed_packages[0] = {}".format(later_matching_fixed_packages[0])) - print( - "\ntype(later_matching_fixed_packages[0]) = {}".format( - type(later_matching_fixed_packages[0]) - ) - ) - # NOTE: This gives us the PURL type but instead we want the PURL itself to pass to assign_univers_version(self, fixed_pkg) and get the command_name in return, which we'll then use in the sort process. - print( - "\nlater_matching_fixed_packages[0].type = {}".format( - later_matching_fixed_packages[0].type - ) - ) - print( - "\ntype(later_matching_fixed_packages[0].type) = {}".format( - type(later_matching_fixed_packages[0].type) - ) - ) - - # Incoming is a list -- later_matching_fixed_packages - # We'll use assign_univers_version() above to get the univers version as a command_name. - # But what do we pass to it? The [0] index of the incoming list, i.e., later_matching_fixed_packages[0]? command_name = self.assign_univers_version(later_matching_fixed_packages[0]) - - print("\n>>> command_name = {}\n".format(command_name)) - - # TODO: Maybe we don't need to convert to a PURL, a list of dictionaries etc.?? - print( - "\n+++++++ later_matching_fixed_packages[0].version = {}".format( - later_matching_fixed_packages[0].version - ) - ) - - # sort test_sort_by_version = [] test_sort_by_version = sorted( - # later_matching_fixed_packages, key=lambda x: versions.DebianVersion(x["version"]) later_matching_fixed_packages, - # key=lambda x: versions.MavenVersion(x.version), key=lambda x: command_name(x.version), ) - print("\ntest_sort_by_version = {}\n".format(test_sort_by_version)) - return test_sort_by_version - # convert_to_dict_list = [] - - # sorted_later_matching_fixed_packages = [] - - # # TODO: First, convert to a list of dictionaries. - # for pkg in later_matching_fixed_packages: - # # pkg is a - # print("pkg = {}".format(pkg)) - # print("type(pkg) = {}".format(type(pkg))) - - # # pkg_str is a string - # pkg_str = pkg.package_url - # print("pkg_str = {}".format(pkg_str)) - # print("type(pkg_str) = {}".format(type(pkg_str))) - - # # purl is a - # purl = PackageURL.from_string(pkg_str) - # print("purl = {}".format(purl)) - # print("type(purl) = {}".format(type(purl))) - - # purl_dict = purl.to_dict() - # print("purl_dict = {}".format(purl_dict)) - # print("type(purl_dict) = {}".format(type(purl_dict))) - - # convert_to_dict_list.append(purl.to_dict()) - # print("HELLO\n") - - # print("\nconvert_to_dict_list = {}\n".format(convert_to_dict_list)) - - # return convert_to_dict_list - # ========================================================== - # sorted_later_matching_fixed_packages = sorted( - # later_matching_fixed_packages, key=lambda x: versions.MavenVersion(x["version"]) - # ) - # print( - # "\nsorted_later_matching_fixed_packages = {}\n".format( - # sorted_later_matching_fixed_packages - # ) - # ) - # print("\n".join(map(str, sorted_later_matching_fixed_packages))) - - # return what? - - # ========================================================== - # ========================================================== - - # def find_closest_fixed_by_package(self, later_matching_fixed_packages): - # # Maybe use sort_by_version() above instead? - # # take the incoming list later_matching_fixed_packages, convert to list of dictionaries, sort by version using univers.version.[version class], choose the top i.e., index [0] and convert back to PURL and return that PURL. - # print("\nlater_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) - - # closest_fixed_by_package = "TBD" - - # return closest_fixed_by_package - - @property - # def get_fixing_packages(self): - def get_closest_fixed_package(self): - """ - This function identifies the closest fixed package version that is greater than the affected package version and - is the same type, namespace, name, qualifiers and subpath as the affected package. - """ - - print("\nself = {}\n".format(self)) - - # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. - # fixed_packages = self.get_fixed_packages(package=self) - # This is clearer. - matching_fixed_packages = self.get_fixed_packages(package=self) - - # This returns a list of the vulnerabilities that affect this package (i.e., self). - qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - - # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). - # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). - qs = qs.prefetch_related( - Prefetch( - "packages", - # queryset=fixed_packages, - queryset=matching_fixed_packages, - # to_attr="filtered_fixed_packages", - to_attr="matching_fixed_packages", - ) - ) - - # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). - print("qs = {}\n".format(qs)) - - # ************************************************************************ - - later_matching_fixed_packages = [] - - vuln_count = 0 - for vuln in qs: - print("vuln = {}\n".format(vuln)) - # print( - # "\tqs[vuln_count].filtered_fixed_packages = {}".format( - # qs[vuln_count].filtered_fixed_packages - # ) - # ) - print( - "\tqs[vuln_count].matching_fixed_packages = {}".format( - qs[vuln_count].matching_fixed_packages - ) - ) - print("") - - # Check the Prefetch qs. - # TODO: Do we want to check whether the fixed version has any vulnerabilities of its own? - # for fixed_pkg in qs[vuln_count].filtered_fixed_packages: - for fixed_pkg in qs[vuln_count].matching_fixed_packages: - print("\tfixed_pkg = {}".format(fixed_pkg)) - print("\tfixed_pkg.type = {}".format(fixed_pkg.type)) - print("\tfixed_pkg.version = {}".format(fixed_pkg.version)) - print("\t--------------------------") - print("\tself.type = {}".format(self.type)) - print("\tself.version = {}".format(self.version)) - - # Assign univers version and compare: False = fixed_pkg.version < self.version (affected version). - - # 2023-08-02 Wednesday 16:01:35. atm immediate_fix is True or False. If instead assign_and_compare_univers_versions() returns the univers version, we could get that here and then test with this or similar right here -- enabling use of the univers version function in other places as well, like a sort_by_version function! - # if command_name(fixed_pkg.version) > command_name(self.version): - # return True - # else: - # return False - # ===================================================== - # Replace this with chunk below - # immediate_fix = self.assign_and_compare_univers_versions(fixed_pkg) - # print("\t--------------------------") - # print("\timmediate_fix = {}\n".format(immediate_fix)) - - # if fixed_pkg in fixed_packages and immediate_fix: - # later_matching_fixed_packages.append(fixed_pkg) - # ===================================================== - # command_name = self.assign_and_compare_univers_versions(fixed_pkg) - # renamed - # TODO: Move this up before the for loop -- both for loops if possible -- to reduce calls! - command_name = self.assign_univers_version(fixed_pkg) - print("\nJust requested command_name >>> {}\n".format(command_name)) - # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( - # self.version - # ): - if fixed_pkg in matching_fixed_packages and command_name( - fixed_pkg.version - ) > command_name(self.version): - later_matching_fixed_packages.append(fixed_pkg) - - vuln_count += 1 - - # find_closest_fixed_by_package -- from the list later_matching_fixed_packages - # closest_fixed_by_package = self.find_closest_fixed_by_package(later_matching_fixed_packages) - - # TODO: or instead use this. This will be a list sorted by univers version class, and here all we need is to grab the [0] index from that list for the closest fixed by package! So we'd return a single closest_fixed_package. - # ALERT: The sort query needs to be done separately for each vulnerability because the list of fixed by packages is likely to be different. As is, we return a single sorted list of all fixed by packages for the affected package and then pass just the [0] package -- not what we want to do! - sort_fixed_by_packages_by_version = self.sort_by_version(later_matching_fixed_packages) - print( - "\nsort_fixed_by_packages_by_version = {}\n".format(sort_fixed_by_packages_by_version) - ) - # ALERT: 2023-08-05 Saturday 23:22:08. Address server error 500? - # ALERT: 2023-08-05 Saturday 23:24:50. This actusally fixed the server error (500) and I can now even see the Packafe details page for pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1 !!! - # HOT: I need to trace back the root cause of the server error (500). I suspect it's something like a record with no fixed by packages or something else that is an empty list but which i try to measure, e.g., for a print statement, or possibly for a real if condition.. - if sort_fixed_by_packages_by_version is None: - return - - closest_fixed_package = sort_fixed_by_packages_by_version[0] - - # print("\n!!! later_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) - # print( - # "\n!!! sort_fixed_by_packages_by_version = {}\n".format( - # sort_fixed_by_packages_by_version - # ) - # ) - # print("\n!!! closest_fixed_package = {}\n".format(closest_fixed_package)) - - # rebuilt_purl_from_dict = PackageURL( - # closest_fixed_package["type"], - # closest_fixed_package["namespace"], - # closest_fixed_package["name"], - # closest_fixed_package["version"], - # closest_fixed_package["qualifiers"], - # closest_fixed_package["subpath"], - # ) - # print("\n!!! rebuilt_purl_from_dict = {}\n".format(rebuilt_purl_from_dict)) - - # return later_matching_fixed_packages - return sort_fixed_by_packages_by_version - # return [closest_fixed_package] - - # return [rebuilt_purl_from_dict] - @property def fixed_package_details(self): """ - This is a test that might develop into a model-based equivalent of the loops etc. I was doing/trying to do in the Jinja2 template. I'm going to add this as a context so we can see it in the template. + Find and report all fixed by packages (and certain related versions) for each vulnerability in + each affected package. Report packages with no vulnerabilities as well. """ - # return "Hello" - - # vcio_dict = { - # [ - # {"VCID-2nyb-8rwu-aaag": "PURL01"}, - # {"VCID-gqhw-ngh8-aaap": "PURL02"}, - # {"some-other-id": "PURL03"}, - # ] - # } - - print("\n==> This is from the test_property_01() property.\n") - - print("\nself = {}\n".format(self)) - - # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. - # fixed_packages = self.get_fixed_packages(package=self) - # This is clearer. + # Get all fixed packages that match the target package (type etc.), regardless of fixed vuln. matching_fixed_packages = self.get_fixed_packages(package=self) - # This returns a list of the vulnerabilities that affect this package (i.e., self). + # Get a list of the vulnerabilities that affect this package (i.e., self). qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - # TODO: Can we get all sibling packages so that we can then determine which have 0 vulnerabilities and of these the closest and maybe the most recent as well? - all_sibling_packages = self.get_sibling_packages(package=self) - print("\nall_sibling_packages = {}\n".format(all_sibling_packages)) - print("\nlen(all_sibling_packages) = {}\n".format(len(all_sibling_packages))) non_vuln_sibs = [] for sib in all_sibling_packages: if sib.is_vulnerable is False: non_vuln_sibs.append(sib) - print("\nnon_vuln_sibs = {}\n".format(non_vuln_sibs)) - print("\nlen(non_vuln_sibs) = {}\n".format(len(non_vuln_sibs))) # Add just the greater-than versions to a new list command_name = self.assign_univers_version(self) - print( - "\nOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO command_name = {}\n".format( - command_name - ) - ) + later_non_vuln_sibs = [] for non_vuln_sib in non_vuln_sibs: if command_name(non_vuln_sib.version) > command_name(self.version): later_non_vuln_sibs.append(non_vuln_sib) - print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) - print("\nlen(later_non_vuln_sibs) = {}\n".format(len(later_non_vuln_sibs))) - - # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). - # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + # Take the list of vulns affecting the current package, retrieve a list of the fixed packages for each vuln, and assign the result to a custom attribute, 'matching_fixed_packages'. Ex: qs[0].matching_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). qs = qs.prefetch_related( Prefetch( "packages", - # queryset=fixed_packages, queryset=matching_fixed_packages, - # to_attr="filtered_fixed_packages", to_attr="matching_fixed_packages", ) ) - # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). - print("\nzzz qs = {}\n".format(qs)) - purl_dict = {} - purl_dict["purl"] = self.purl - purl_dict.update({"vulnerabilities": []}) - # purl_dict["vulnerabilities"].append({"fruit": "orange"}) + # HOT: In constructing the purl_dict, I need to include packages that have no vulnerabilities -- we still want various dict fields to be populated to reflect the relevant structure and content all such packages. atm only a few fields are included in the dict for such a package. "closest_non_vulnerable_fix", "closest_non_vulnerable_fix_url", "most_recent_non_vulnerable_fix" and "most_recent_non_vulnerable_fix_url" are currently omitted. for vuln in qs: - print("\nzzz vuln = {}\n".format(vuln)) - print("\nzzz type(vuln) = {}\n".format(type(vuln))) - later_matching_fixed_packages = [] - # purl_dict[vuln.vulnerability_id] = "aaa" - # purl_dict.update({"vulnerability": vuln.vulnerability_id}) - purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) - - # TODO:2023-08-05 Saturday 13:12:28. This returns a list of matching fixed packages for this specific vuln! vuln_matching_fixed_packages = vuln.matching_fixed_packages - print("\nzzz self.purl = {}\n".format(self.purl)) - print("\nzzz vuln = {}\n".format(vuln)) - print("\nzzz vuln_matching_fixed_packages = {}\n".format(vuln_matching_fixed_packages)) - - # TODO: So we need to sort this list by version using the correct univers version and then return the [0] index in that sorted list - # QUESTION: Do we still need to remove lesser-than fixed packages or did we already do that? - - # ============================================================= - # command_name = self.assign_univers_version(fixed_pkg) command_name = self.assign_univers_version(self) - print("\nzzz command_name = {}\n".format(command_name)) - - # ALERT: What if there are no fixed by packages? The following thows an error because the list 'vuln_matching_fixed_packages' is empty! - # [I fixed this, right? ;-] - closest_fixed_package = "" if len(vuln_matching_fixed_packages) > 0: - for fixed_pkg in vuln_matching_fixed_packages: if fixed_pkg in matching_fixed_packages and command_name( fixed_pkg.version ) > command_name(self.version): later_matching_fixed_packages.append(fixed_pkg) - # print("\nJust requested command_name >>> {}\n".format(command_name)) - # # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( - # # self.version - # # ): - # if fixed_pkg in matching_fixed_packages and command_name( - # fixed_pkg.version - # ) > command_name(self.version): - # later_matching_fixed_packages.append(fixed_pkg) - # ============================================================= - # later_matching_fixed_packages = vuln.matching_fixed_packages - - print( - "\nzzz later_matching_fixed_packages = {}\n".format( - later_matching_fixed_packages - ) - ) - sort_fixed_by_packages_by_version = self.sort_by_version( later_matching_fixed_packages ) - print( - "\nzzz sort_fixed_by_packages_by_version = {}\n".format( - sort_fixed_by_packages_by_version - ) - ) + closest_fixed_package = sort_fixed_by_packages_by_version[0] - # closest_fixed_package = sort_fixed_by_packages_by_version[0].purl - # 2023-08-06 Sunday 11:15:03. This returns a queryset of vulns affecting this package. - # HOT: How do we get the closest fixed by package vuln count and list of vulns? I keep getting errors. ;-) closest_fixed_package_vulns = closest_fixed_package.affected_by - # closest_fixed_package_vulns_list = list(closest_fixed_package_vulns) - # ALERT: 2023-08-06 Sunday 14:25:49. This did the trick! - - # closest_fixed_package_vulns_list = [ - # i.vulnerability_id for i in closest_fixed_package_vulns - # ] - - # 2023-08-06 Sunday 16:53:27. Try a named tuple to pass the vuln's vulnerability+id and get_absolute_url. - # FixedPackageVuln = namedtuple("FixedPackageVuln", "vuln_id, vuln_get_absolute_url") - # closest_fixed_package_vulns_list = [ - # FixedPackageVuln( - # vuln_id=fixed_pkg_vuln.vulnerability_id, - # vuln_get_absolute_url=fixed_pkg_vuln.get_absolute_url(), - # ) - # for fixed_pkg_vuln in closest_fixed_package_vulns - # ] - # ALERT: Replace the namedtuple with a dict -- this way it can be added to the purl_dict as a nested dict rather than a list of 2 values. + closest_fixed_package_vulns_dict = [ { "vuln_id": fixed_pkg_vuln.vulnerability_id, @@ -1109,127 +774,25 @@ def fixed_package_details(self): for fixed_pkg_vuln in closest_fixed_package_vulns ] - # === - # closest_fixed_package_vulns_list = closest_fixed_package_vulns.objects.values_list() - - # closest_fixed_package_vulns_list = [] - # for closest_vuln in closest_fixed_package_vulns: - # closest_fixed_package_vulns_list.append(closest_vuln) - # print( - # "\t\nQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ vuln = {}\n".format( - # closest_vuln - # ) - # ) - # print("\t\ntype(closest_vuln) = {}".format(type(closest_vuln))) - - # # closest_fixed_package_vuln_count = len(closest_fixed_package.affected_by) - # print( - # "\t\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type(closest_fixed_package_vulns) = {}\n".format( - # type(closest_fixed_package_vulns) - # ) - # ) - else: closest_fixed_package = "There are no reported fixed packages." - # Is None the value we want? We do not want to display anything but the count = 0. - # closest_fixed_package_vulns = None - # closest_fixed_package_vuln_count = 0 - - # closest_fixed_package_vulns_list = [] - - print("\nzzz closest_fixed_package = {}".format(closest_fixed_package)) - print("zzz type(closest_fixed_package) = {}\n".format(type(closest_fixed_package))) - - # TODO: How do we add 'closest_fixed_by_purl', 'closest_fixed_by_vulns' and 'non_vulnerable_fix'? - - # # for vuln in purl_dict["vulnerabilities"]: - # # # vuln["closest_fixed_by_purl"] = "?????" - # # vuln["closest_fixed_by_purl"] = closest_fixed_package - # # vuln["closest_fixed_by_url"] = "?????" - # # vuln["closest_fixed_by_vulnerabilities"] = "?????" - # # vuln["non_vulnerable_fix"] = "?????" - # # vuln["non_vulnerable_fix_url"] = "?????" for dict_vuln in purl_dict["vulnerabilities"]: - print("\n===================================> vuln = {}\n".format(vuln)) - print("\n===================================> type(vuln) = {}\n".format(type(vuln))) - print("\n===================================> vuln.vcid = {}\n".format(vuln.vcid)) - print( - "\n===================================> dict_vuln['vulnerability'] = {}\n".format( - dict_vuln["vulnerability"] - ) - ) - # TODO: Up above we defined 'non_vuln_sibs' but we still need to remove those with less than version - # ALERT: remove less than versions from 'non_vuln_sibs' - # 2023-08-05 Saturday 20:30:47. Hopefully just wrote the code for that up above, with the new list 'later_non_vuln_sibs'. - closest_non_vulnerable_fix = "" - # if len(non_vuln_sibs) > 0: - # closest_non_vulnerable_fix = self.sort_by_version(non_vuln_sibs)[0] + if len(later_non_vuln_sibs) > 0: closest_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[0] - # else: - # # closest_non_vulnerable_fix = ( - # # "There are no reported non-vulnerable fixed packages." - # # ) - # closest_non_vulnerable_fix = None most_recent_non_vulnerable_fix = "" if len(later_non_vuln_sibs) > 0: most_recent_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[-1] else: - # most_recent_non_vulnerable_fix = ( - # "There are no reported non-vulnerable fixed packages." - # ) most_recent_non_vulnerable_fix = None - # if dict_vuln["vulnerability"] == vuln.vulnerability_id: if dict_vuln["vulnerability"] == str(vuln): - # if dict_vuln["vulnerability"] == vuln.vcid: - # dict_vuln["closest_fixed_by_purl"] = "?????" dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() - # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vuln_count - # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns - # dict_vuln["closest_fixed_by_vulnerabilities"] = ["A", "B"] - - # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_list - # ALERT: Replace the above list created with a namedtuple with the following dictionary: dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_dict - # ALERT: Moved these up 1 level in the dict. - # dict_vuln["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) - # dict_vuln[ - # "closest_non_vulnerable_fix_url" - # ] = closest_non_vulnerable_fix.get_absolute_url() - # dict_vuln["most_recent_non_vulnerable_fix"] = str( - # most_recent_non_vulnerable_fix - # ) - # dict_vuln[ - # "most_recent_non_vulnerable_fix_url" - # ] = most_recent_non_vulnerable_fix.get_absolute_url() - - # QUESTION: Can we add the non-vuln data as higher-level key-value pairs rather than children of "vulnerabilities"? - - # purl_dict.update({"fruits": []}) - # purl_dict["fruits"].append({"fruit": "apple"}) - # purl_dict["fruits"].append({"fruit": "banana"}) - - # purl_dict.update( - # {"closest_non_vulnerable_fix": str(closest_non_vulnerable_fix)} - # ) - # purl_dict.update( - # { - # "closest_non_vulnerable_fix_url": closest_non_vulnerable_fix.get_absolute_url() - # } - # ) - # purl_dict.update( - # {"most_recent_non_vulnerable_fix": str(most_recent_non_vulnerable_fix)} - # ) - # purl_dict.update( - # { - # "most_recent_non_vulnerable_fix_url": most_recent_non_vulnerable_fix.get_absolute_url() - # } - # ) purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) purl_dict[ @@ -1242,76 +805,14 @@ def fixed_package_details(self): "most_recent_non_vulnerable_fix_url" ] = most_recent_non_vulnerable_fix.get_absolute_url() - print("\npurl_dict = {}\n".format(purl_dict)) - - print(json.dumps(purl_dict, indent=4, sort_keys=False)) - - # # Print to text file - pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) - # logger = logging.getLogger(__name__) - # logger.setLevel(logging.INFO) - # # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt")) - # # will this overwrite prior writes? weird output - # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt", mode="w")) - # logger.info(pretty_purl_dict) - - with open("/home/jmh/pretty_purl_dict.txt", "w") as f: - f.write(pretty_purl_dict) - - alternate_dict_01 = { - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", - "vulnerabilities": [ - { - "vulnerability": "VCID-2nyb-8rwu-aaag", - "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - # "get_absolute_url": reverse("package_details", args=[self.purl]), - "closest_fixed_by_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2"], - ), - "closest_fixed_by_vulns": 2, - "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "non_vulnerable_fix_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], - ), - }, - { - "vulnerability": "VCID-gqhw-ngh8-aaap", - "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4", - # "get_absolute_url": reverse("package_details", args=[self.purl]), - "closest_fixed_by_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"], - ), - "closest_fixed_by_vulns": 1, - "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "non_vulnerable_fix_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], - ), - }, - { - "vulnerability": "VCID-t7e4-g3fr-aaan", - "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - # "get_absolute_url": reverse("package_details", args=[self.purl]), - "closest_fixed_by_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], - ), - "closest_fixed_by_vulns": 0, - "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "non_vulnerable_fix_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], - ), - }, - ], - } - - # return vcio_dict + # Temporary print output during dev/testing: + # print("\npurl_dict = {}\n".format(purl_dict)) + # print(json.dumps(purl_dict, indent=4, sort_keys=False)) - # return alternate_dict_01 + # TODO: Consider whether we want to provide the user with an option to output the dictionary to a file. + # pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) + # with open("pretty_purl_dict.txt", "w") as f: + # f.write(pretty_purl_dict) return purl_dict diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 8d204bccf..236379ab6 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -20,7 +20,7 @@ -
+
@@ -40,18 +40,47 @@
+ {% if affected_by_vulnerabilities|length != 0 %} + +
+ + + + + + + + + + + +
+ Closest non-vulnerable purl + + {{ fixed_package_details.closest_non_vulnerable_fix }} +
+ Latest non-vulnerable purl + + {{ fixed_package_details.most_recent_non_vulnerable_fix }} +
+
+ + {% endif %} +
Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }})
+ + - + @@ -75,159 +104,63 @@ {% endif %} {% endfor %} - {% empty %} - + {% endfor %} -
Vulnerability Summary AliasesFixed by packagesClosest fixed by packages
- - - - - - - - {% if package.purl in fixed_package_details.purl %} - {% for key, value in fixed_package_details.items %} - {% if key == "vulnerabilities" %} + + {% if package.purl in fixed_package_details.purl %} + {% for key, value in fixed_package_details.items %} + {% if key == "vulnerabilities" %} {% for abc in value %} {% if abc.vulnerability == vulnerability.vulnerability_id %} -
    -
  • - PURL: {{ fixed_package_details.purl }} - - - -
    • -
  • - Vulnerability: {{ abc.vulnerability }} -
    • -
  • - Closest fixed-by PURL: - {{ abc.closest_fixed_by_purl }} -
    • -
  • - Closest fixed-by vulnerability count: {{ abc.closest_fixed_by_vulnerabilities|length }} - {% if abc.closest_fixed_by_vulnerabilities|length != 0 %} -
    -
    - -
    - -
    - {% endif %} -
    • -
  • - Closest non-vulnerable fix: - - - {{ fixed_package_details.closest_non_vulnerable_fix }} -
    • -
  • - Most recent non-vulnerable fix: - - - {{ fixed_package_details.most_recent_non_vulnerable_fix }} -
    • -
{% endif %} {% endfor %} - - - {% endif %} - {% endfor %} - {% else %} - NO-- {{ package.purl }} {% endif %} + {% endfor %} - - - - - - + {% endif %}
This package is not known to be affected by vulnerabilities.
@@ -244,7 +177,6 @@ Aliases - {% for vulnerability in fixing_vulnerabilities %} diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 24a66d2d2..3535173ee 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -96,60 +96,43 @@ def test_vulnerability_package(self): class TestPackageModel(TestCase): def test_univers_version_comparisons(self): assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") - assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") # pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 is a real PURL in the DB - # But I get an error when I try to compare 2 PURLs with the same suffix -- - # univers.versions.InvalidVersion: '2.12.1-1%2Bdeb11u1' is not a valid - # Do we need to replace/delete the "%"? - # assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( - # "2.13.1-1%2Bdeb11u1" - # ) - # Test the error + # But we need to replace/delete the "%". Test the error: with pytest.raises(versions.InvalidVersion): assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( "2.13.1-1%2Bdeb11u1" ) - # Decode the version and test. + # Decode the version and test: assert versions.DebianVersion( urllib.parse.unquote("2.12.1-1%2Bdeb11u1") ) < versions.DebianVersion(urllib.parse.unquote("2.13.1-1%2Bdeb11u1")) + # Expect an error when comparing different types. with pytest.raises(TypeError): assert versions.PypiVersion("0.9") < versions.DebianVersion("0.10") - # Using versions.Version does not correctly make this comparison! + # This demonstrates that versions.Version does not correctly compare 0.9 vs. 0.10. assert not versions.Version("0.9") < versions.Version("0.10") # Use SemverVersion instead as a default fallback version for comparisons. assert versions.SemverVersion("0.9") < versions.SemverVersion("0.10") - def test_assign_and_compare_univers_versions(self): deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") + assert versions.DebianVersion(deb01.version) < versions.DebianVersion(deb02.version) - immediate_fix01 = deb01.assign_and_compare_univers_versions(deb02) - print("\nimmediate_fix01 = {}\n".format(immediate_fix01)) - # assert deb01.assign_and_compare_univers_versions(deb02) is True - assert deb01.assign_and_compare_univers_versions(deb02) + def test_assign_univers_version(self): + requesting_package = models.Package.objects.create(type="deb", name="git", version="2.30.1") - immediate_fix02 = deb02.assign_and_compare_univers_versions(deb01) - print("\nimmediate_fix02 = {}\n".format(immediate_fix02)) - # assert deb02.assign_and_compare_univers_versions(deb01) is False - assert not deb02.assign_and_compare_univers_versions(deb01) + deb01 = models.Package.objects.create(type="deb", name="git", version="2.31.1") + command_name_deb01 = requesting_package.assign_univers_version(deb01) + assert command_name_deb01 == versions.DebianVersion pypi01 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") - pypi02 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.10") - - immediate_fix03 = pypi01.assign_and_compare_univers_versions(pypi02) - print("\nimmediate_fix03 = {}\n".format(immediate_fix03)) - # assert pypi01.assign_and_compare_univers_versions(pypi02) is True - assert pypi01.assign_and_compare_univers_versions(pypi02) + command_name_pypi01 = requesting_package.assign_univers_version(pypi01) + assert command_name_pypi01 == versions.PypiVersion gem01 = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") - gem02 = models.Package.objects.create(type="gem", name="sidekiq", version="0.10") - - immediate_fix04 = gem01.assign_and_compare_univers_versions(gem02) - print("\nimmediate_fix04 = {}\n".format(immediate_fix04)) - # assert gem01.assign_and_compare_univers_versions(gem02) is True - assert gem01.assign_and_compare_univers_versions(gem02) + command_name_gem01 = requesting_package.assign_univers_version(gem01) + assert command_name_gem01 == versions.SemverVersion diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 396ac1bc2..41ecb1455 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -83,9 +83,6 @@ def get_context_data(self, **kwargs): context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) - # context["get_fixing_packages"] = package.get_fixing_packages - context["get_closest_fixed_package"] = package.get_closest_fixed_package - # context["test_property_01"] = package.test_property_01 context["fixed_package_details"] = package.fixed_package_details return context diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 0071686fb..3187b67ec 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -39,9 +39,7 @@ CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS", default=[]) # SECURITY WARNING: do not run with debug turned on in production -# DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) -# DEBUG = "127.0.0.1" -DEBUG = True +DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) # SECURITY WARNING: do not run with debug turned on in production DEBUG_TOOLBAR = env.bool("VULNERABLECODE_DEBUG_TOOLBAR", default=False) diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index 96ef5aad3..38da36039 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -198,16 +198,14 @@ code { } .two-col-left { - width: 160px; + width: 250px; text-align: right !important; font-weight: bold; - padding-right: 20px !important; + padding-right: 15px !important; line-height: 20px; - border: solid 1px #e8e8e8 !important; } .two-col-right { - border: solid 1px #e8e8e8 !important; line-height: 20px; } @@ -378,85 +376,6 @@ span.tag.custom { margin: 0px 0px 6px 10px; } -/* test bulleted list */ - -ul.fixed_by_bullet { - list-style-type: disc; - /*margin-top: 2px; -margin-bottom: 10px;*/ - /*margin-left: -24px;*/ - /*margin-left: -30px;*/ - margin-top: 0.25em; - margin-left: 7px; - margin-bottom: 0.25em; - padding-left: 10px; -} - -ul.fixed_by_bullet ul { - list-style-type: disc; - /*margin-top: 10px;*/ - margin-top: 5px; - margin-top: 0px; - margin-bottom: 0px; - margin-left: 23px; - margin-left: 18px; - padding: 0; - border: none; -} - - - -ul.fixed_by_bullet li { - margin-left: 0px; - font-family: Arial; - font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; - font-size: 13px; - font-weight: normal; - /*margin-bottom: 10px;*/ - margin-bottom: 2px; -} - -ul.fixed_by_bullet li:last-child { - margin-left: 0px; - font-family: Arial; - font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; - font-size: 13px; - font-weight: normal; - /*margin-bottom: 10px;*/ - margin-bottom: 0px; -} - -ul.fixed_by_bullet li li { - margin-left: 0px; - font-family: Arial; - font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; - font-size: 13px; - font-weight: normal; - margin-top: 0px; - color: #000000; -} - -/* 10/10/15 add 3rd-level bullets */ -ul.fixed_by_bullet ul ul { - list-style-type: disc; - margin-top: 0px; - margin-bottom: 0px; - margin-left: 50px; - margin-left: 17px; - padding: 0; - border: none; -} - -ul.fixed_by_bullet li li li { - margin-left: 0px; - font-family: Arial; - font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; - font-size: 13px; - font-weight: normal; - margin-top: 0px; - color: #000000; -} - /* CSS for dev fixed by headers */ .dev_fixed_by_headers { border: solid 1px #cccccc; @@ -469,3 +388,27 @@ ul.fixed_by_bullet li li li { margin-bottom: 3px; display: block; } + +.non-floating-purl { + position: relative; + width: 100%; + z-index: 100; + margin-bottom: 0px; +} + +.non-floating-purl .table td, +.non-floating-purl .table tbody tr:last-child td, +.non-floating-purl .table th { + border: solid 1px #dbdbdb; + background-color: #ffffff; +} + +.non-vuln { + margin-top: -25px; +} + +.non-vuln .table td, +.non-vuln .table tbody tr:last-child td, +.non-vuln .table th { + border: solid 1px #dbdbdb; +} From 814cd063bc10e533de5bb027149ff59ffe1364f3 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 14 Aug 2023 15:04:38 -0700 Subject: [PATCH 06/53] Add univers version, revise sort and related code, update and add new tests #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 117 +++++------ vulnerabilities/tests/test_models.py | 292 +++++++++++++++++++++++++-- 2 files changed, 326 insertions(+), 83 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index b8b2bea94..9b46cff97 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -33,6 +33,7 @@ from packageurl.contrib.django.models import without_empty_values from rest_framework.authtoken.models import Token from univers import versions +from univers.version_range import RANGE_CLASS_BY_SCHEMES from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage @@ -648,60 +649,16 @@ def get_sibling_packages(self, package): subpath=package.subpath, ).distinct() - def assign_univers_version(self, fixed_pkg): - """ - Identify which univers version applies to a package and return that version for use, e.g., - in sorting a group of sibling packages (same type etc.). - """ - # TODO: Many more to be added. - match_type_to_univers_version = { - "conan": versions.ConanVersion, - "deb": versions.DebianVersion, - # The following throws an error: AttributeError: module 'univers.versions' has no attribute 'GemVersion' - # "gem": versions.GemVersion, - "maven": versions.MavenVersion, - "nginx": versions.NginxVersion, - # "npm": - "openssl": versions.OpensslVersion, - "pypi": versions.PypiVersion, - # from https://github.com/nexB/univers/blob/205da7ecbf7b0f195662373ea710b2b84a877eb0/tests/test_version_comparison.py - # versions.SemverVersion, - # versions.GolangVersion, - # versions.PypiVersion, - # versions.GenericVersion, - # versions.ComposerVersion, - # versions.NginxVersion, - # versions.ArchLinuxVersion, - # versions.DebianVersion, - # versions.RpmVersion, - # versions.MavenVersion, - # versions.NugetVersion, - # versions.GentooVersion, - # versions.OpensslVersion, - # versions.LegacyOpensslVersion, - # versions.AlpineLinuxVersion, - } - - command_name = "" - matched_type_to_version = match_type_to_univers_version.get(fixed_pkg.type) - if matched_type_to_version: - command_name = matched_type_to_version - else: - command_name = versions.SemverVersion - - return command_name - - def sort_by_version(self, later_matching_fixed_packages): + def sort_by_version(self, packages_to_sort): # Incoming is a list of - # We'll use assign_univers_version() above to get the univers version as a command_name. - command_name = self.assign_univers_version(later_matching_fixed_packages[0]) - test_sort_by_version = [] - test_sort_by_version = sorted( - later_matching_fixed_packages, - key=lambda x: command_name(x.version), + univers_version = RANGE_CLASS_BY_SCHEMES[packages_to_sort[0].type].version_class + sorted_by_version = [] + sorted_by_version = sorted( + packages_to_sort, + key=lambda x: univers_version(x.version), ) - return test_sort_by_version + return sorted_by_version @property def fixed_package_details(self): @@ -723,11 +680,10 @@ def fixed_package_details(self): non_vuln_sibs.append(sib) # Add just the greater-than versions to a new list - command_name = self.assign_univers_version(self) - + univers_version = RANGE_CLASS_BY_SCHEMES[self.type].version_class later_non_vuln_sibs = [] for non_vuln_sib in non_vuln_sibs: - if command_name(non_vuln_sib.version) > command_name(self.version): + if univers_version(non_vuln_sib.version) > univers_version(self.version): later_non_vuln_sibs.append(non_vuln_sib) # Take the list of vulns affecting the current package, retrieve a list of the fixed packages for each vuln, and assign the result to a custom attribute, 'matching_fixed_packages'. Ex: qs[0].matching_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). @@ -743,20 +699,24 @@ def fixed_package_details(self): purl_dict["purl"] = self.purl purl_dict.update({"vulnerabilities": []}) - # HOT: In constructing the purl_dict, I need to include packages that have no vulnerabilities -- we still want various dict fields to be populated to reflect the relevant structure and content all such packages. atm only a few fields are included in the dict for such a package. "closest_non_vulnerable_fix", "closest_non_vulnerable_fix_url", "most_recent_non_vulnerable_fix" and "most_recent_non_vulnerable_fix_url" are currently omitted. + purl_dict["closest_non_vulnerable_fix"] = "" + purl_dict["closest_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix"] = "" + purl_dict["most_recent_non_vulnerable_fix_url"] = "" for vuln in qs: later_matching_fixed_packages = [] purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) vuln_matching_fixed_packages = vuln.matching_fixed_packages - command_name = self.assign_univers_version(self) closest_fixed_package = "" + # Need to define here to avoid "" vs. [] for some closest_fixed_by_vulnerabilities values? + closest_fixed_package_vulns_dict = [] if len(vuln_matching_fixed_packages) > 0: for fixed_pkg in vuln_matching_fixed_packages: - if fixed_pkg in matching_fixed_packages and command_name( + if fixed_pkg in matching_fixed_packages and univers_version( fixed_pkg.version - ) > command_name(self.version): + ) > univers_version(self.version): later_matching_fixed_packages.append(fixed_pkg) sort_fixed_by_packages_by_version = self.sort_by_version( @@ -765,7 +725,7 @@ def fixed_package_details(self): closest_fixed_package = sort_fixed_by_packages_by_version[0] closest_fixed_package_vulns = closest_fixed_package.affected_by - + # 2023-08-13 Sunday 16:25:41. Not sure but I think I need to define this initially above just after closest_fixed_package = "" closest_fixed_package_vulns_dict = [ { "vuln_id": fixed_pkg_vuln.vulnerability_id, @@ -780,6 +740,7 @@ def fixed_package_details(self): for dict_vuln in purl_dict["vulnerabilities"]: closest_non_vulnerable_fix = "" + # TODO: 2023-08-13 Sunday 16:01:57. Refactor here and elsewhere! if len(later_non_vuln_sibs) > 0: closest_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[0] @@ -787,27 +748,45 @@ def fixed_package_details(self): if len(later_non_vuln_sibs) > 0: most_recent_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[-1] else: - most_recent_non_vulnerable_fix = None + most_recent_non_vulnerable_fix = "" if dict_vuln["vulnerability"] == str(vuln): dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) - dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() - dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_dict + if len(vuln_matching_fixed_packages) > 0: + dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() + closest_fixed_package_vulns_dict = [ + { + "vuln_id": fixed_pkg_vuln.vulnerability_id, + "vuln_get_absolute_url": fixed_pkg_vuln.get_absolute_url(), + } + for fixed_pkg_vuln in closest_fixed_package_vulns + ] + dict_vuln[ + "closest_fixed_by_vulnerabilities" + ] = closest_fixed_package_vulns_dict + else: + dict_vuln["closest_fixed_by_url"] = "" + dict_vuln["closest_fixed_by_vulnerabilities"] = [] purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) - purl_dict[ - "closest_non_vulnerable_fix_url" - ] = closest_non_vulnerable_fix.get_absolute_url() + if len(vuln_matching_fixed_packages) > 0: + purl_dict[ + "closest_non_vulnerable_fix_url" + ] = closest_non_vulnerable_fix.get_absolute_url() + purl_dict[ + "most_recent_non_vulnerable_fix_url" + ] = most_recent_non_vulnerable_fix.get_absolute_url() + else: + purl_dict["closest_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix"] = str( most_recent_non_vulnerable_fix ) - purl_dict[ - "most_recent_non_vulnerable_fix_url" - ] = most_recent_non_vulnerable_fix.get_absolute_url() # Temporary print output during dev/testing: # print("\npurl_dict = {}\n".format(purl_dict)) - # print(json.dumps(purl_dict, indent=4, sort_keys=False)) + print(json.dumps(purl_dict, indent=4, sort_keys=False)) # TODO: Consider whether we want to provide the user with an option to output the dictionary to a file. # pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 3535173ee..59b08dd79 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -10,13 +10,19 @@ import urllib.parse from datetime import datetime from unittest import TestCase +from unittest import mock import pytest +from django.db import transaction +from django.db.models.query import QuerySet from django.db.utils import IntegrityError from freezegun import freeze_time +from packageurl import PackageURL from univers import versions +from univers.version_range import RANGE_CLASS_BY_SCHEMES from vulnerabilities import models +from vulnerabilities.models import Package class TestVulnerabilityModel(TestCase): @@ -94,10 +100,156 @@ def test_vulnerability_package(self): @pytest.mark.django_db class TestPackageModel(TestCase): + def setUp(self): + self.vuln1 = models.Vulnerability.objects.create( + summary="test-vuln1", + vulnerability_id="VCID-123", + ) + self.vuln2 = models.Vulnerability.objects.create( + summary="test-vuln2", + vulnerability_id="VCID-456", + ) + + # Create a vuln of its own for the fixed_by_package + self.vuln3 = models.Vulnerability.objects.create( + summary="test-vuln-not-used-anywhere", + vulnerability_id="VCID-000", + ) + + self.vulnerablecode_package = models.Package.objects.create( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-databind", + version="2.13.1", + qualifiers={}, + subpath="", + ) + + self.fixed_by_package = models.Package.objects.create( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-databind", + version="2.13.2", + qualifiers={}, + subpath="", + ) + + self.backport_fixed_by_package = models.Package.objects.create( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-databind", + version="2.12.6.1", + qualifiers={}, + subpath="", + ) + + self.non_vulnerable_package = models.Package.objects.create( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-databind", + version="2.14.0-rc1", + qualifiers={}, + subpath="", + ) + + models.PackageRelatedVulnerability.objects.create( + package=self.vulnerablecode_package, + vulnerability=self.vuln1, + fix=False, + ) + + models.PackageRelatedVulnerability.objects.create( + package=self.vulnerablecode_package, + vulnerability=self.vuln2, + fix=False, + ) + + # Create a fixed_by package for vuln1 + models.PackageRelatedVulnerability.objects.create( + package=self.fixed_by_package, + vulnerability=self.vuln1, + fix=True, + ) + + # Add backport_fixed_by_package as a fixed_by for vuln1 -- but this should be excluded because its version is less than the affected package's version. + models.PackageRelatedVulnerability.objects.create( + package=self.backport_fixed_by_package, + vulnerability=self.vuln1, + fix=True, + ) + + # Create a vuln of its own for the fixed_by_packagefixed_by package for vuln1 + models.PackageRelatedVulnerability.objects.create( + package=self.fixed_by_package, + vulnerability=self.vuln3, + fix=False, + ) + + def test_get_vulnerable_packages(self): + vuln_packages = Package.objects.vulnerable() + assert vuln_packages.count() == 3 + assert vuln_packages.distinct().count() == 2 + + # matching_fixed_packages = vulnerablecode_package.get_fixed_packages(vulnerablecode_package) + # assert vuln_packages.distinct()[0] + + # matching_fixed_packages = vuln_packages.distinct()[0].get_fixed_packages( + # vuln_packages.distinct()[0] + # ) + + first_vulnerable_package = vuln_packages.distinct()[0] + matching_fixed_packages = first_vulnerable_package.get_fixed_packages( + first_vulnerable_package + ) + first_fixed_by_package = matching_fixed_packages[0] + + assert ( + first_vulnerable_package.purl + == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" + ) + assert len(matching_fixed_packages) == 2 + assert ( + first_fixed_by_package.purl + == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1" + ) + + purl_dict = { + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "vulnerabilities": [ + { + "vulnerability": "VCID-123", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "closest_fixed_by_url": "/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "closest_fixed_by_vulnerabilities": [ + { + "vuln_id": "VCID-000", + "vuln_get_absolute_url": "/vulnerabilities/VCID-000", + } + ], + }, + { + "vulnerability": "VCID-456", + "closest_fixed_by_purl": "There are no reported fixed packages.", + "closest_fixed_by_url": "", + "closest_fixed_by_vulnerabilities": [], + }, + ], + "closest_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "closest_non_vulnerable_fix_url": "", + "most_recent_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "most_recent_non_vulnerable_fix_url": "", + } + + assert vuln_packages.distinct()[0].fixed_package_details == purl_dict + def test_univers_version_comparisons(self): assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") + deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") + deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") + assert versions.DebianVersion(deb01.version) < versions.DebianVersion(deb02.version) + # pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 is a real PURL in the DB # But we need to replace/delete the "%". Test the error: with pytest.raises(versions.InvalidVersion): @@ -118,21 +270,133 @@ def test_univers_version_comparisons(self): # Use SemverVersion instead as a default fallback version for comparisons. assert versions.SemverVersion("0.9") < versions.SemverVersion("0.10") - deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") - deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") - assert versions.DebianVersion(deb01.version) < versions.DebianVersion(deb02.version) + def test_univers_version_class(self): + gem_version = RANGE_CLASS_BY_SCHEMES["gem"].version_class + assert gem_version == versions.RubygemsVersion + + gem_package = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") + gem_package_version = RANGE_CLASS_BY_SCHEMES[gem_package.type].version_class + assert gem_package_version == versions.RubygemsVersion + + deb_version = RANGE_CLASS_BY_SCHEMES["deb"].version_class + assert deb_version == versions.DebianVersion + + deb_package = models.Package.objects.create(type="deb", name="git", version="2.31.1") + deb_package_version = RANGE_CLASS_BY_SCHEMES[deb_package.type].version_class + assert deb_package_version == versions.DebianVersion + + pypi_version = RANGE_CLASS_BY_SCHEMES["pypi"].version_class + assert pypi_version == versions.PypiVersion + + pypi_package = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") + pypi_package_version = RANGE_CLASS_BY_SCHEMES[pypi_package.type].version_class + assert pypi_package_version == versions.PypiVersion + + def test_sort_by_version(self): + list_to_sort = [ + "pkg:npm/sequelize@3.13.1", + "pkg:npm/sequelize@3.10.1", + "pkg:npm/sequelize@3.40.1", + "pkg:npm/sequelize@3.9.1", + ] + + # Convert list of strings ^ to a list of vulnerablecode Package objects. + vuln_pkg_list = [] + for package in list_to_sort: + purl = PackageURL.from_string(package) + attrs = {k: v for k, v in purl.to_dict().items() if v} + vulnerablecode_package = models.Package.objects.create(**attrs) + vuln_pkg_list.append(vulnerablecode_package) - def test_assign_univers_version(self): - requesting_package = models.Package.objects.create(type="deb", name="git", version="2.30.1") + requesting_package = models.Package.objects.create( + type="npm", + name="sequelize", + version="3.0.0", + ) + + sorted_pkgs = requesting_package.sort_by_version(vuln_pkg_list) + first_sorted_item = sorted_pkgs[0] + + assert sorted_pkgs[0].purl == "pkg:npm/sequelize@3.9.1" + assert sorted_pkgs[-1].purl == "pkg:npm/sequelize@3.40.1" + + def test_string_to_purl_to_dict_to_package(self): + # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., + # a . + + # Convert a PURL string to a PURL. + purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + purl = PackageURL.from_string(purl_string) + + assert type(purl) == PackageURL + assert purl.type == "maven" + assert purl.qualifiers == {} + assert purl.subpath == None + + # Convert the PURL to a dictionary. + # It appears that this step is where the unwanted None values are created for qualifiers and + # subpath when the PURL does not already contain values for those attributes. + purl_to_dict = purl.to_dict() + + assert purl_to_dict == { + "type": "maven", + "namespace": "org.apache.tomcat.embed", + "name": "tomcat-embed-core", + "version": "9.0.31", + "qualifiers": None, + "subpath": None, + } + assert purl_to_dict.get("qualifiers") == None + assert purl_to_dict.get("subpath") == None - deb01 = models.Package.objects.create(type="deb", name="git", version="2.31.1") - command_name_deb01 = requesting_package.assign_univers_version(deb01) - assert command_name_deb01 == versions.DebianVersion + # Convert the dictionary to a VulnerableCode Package, i.e., + # a - pypi01 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") - command_name_pypi01 = requesting_package.assign_univers_version(pypi01) - assert command_name_pypi01 == versions.PypiVersion + # If subpath is None we get error: django.db.utils.IntegrityError: null value in column + # "subpath" violates not-null constraint -- need to convert value from None to empty string. + # Similar issue with qualifiers, which must be converted from None to {}. - gem01 = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") - command_name_gem01 = requesting_package.assign_univers_version(gem01) - assert command_name_gem01 == versions.SemverVersion + # I've structured the following in this way because trying instead to use + # "with pytest.raises(IntegrityError):" will throw the error + # django.db.transaction.TransactionManagementError: An error occurred in the current + # transaction. You can't execute queries until the end of the 'atomic' block. + + try: + with transaction.atomic(): + vulnerablecode_package = models.Package.objects.create( + type=purl_to_dict.get("type"), + namespace=purl_to_dict.get("namespace"), + name=purl_to_dict.get("name"), + version=purl_to_dict.get("version"), + qualifiers=purl_to_dict.get("qualifiers"), + subpath=purl_to_dict.get("subpath"), + ) + except IntegrityError: + print("\nAs expected, an IntegrityError has occurred.\n") + + # This will avoid the IntegrityError: + if purl_to_dict.get("qualifiers") is None: + purl_to_dict["qualifiers"] = {} + if purl_to_dict.get("subpath") is None: + purl_to_dict["subpath"] = "" + + # Check the qualifiers and subpath values again. + assert purl_to_dict.get("qualifiers") == {} + assert purl_to_dict.get("subpath") == "" + + vulnerablecode_package = models.Package.objects.create( + type=purl_to_dict.get("type"), + namespace=purl_to_dict.get("namespace"), + name=purl_to_dict.get("name"), + version=purl_to_dict.get("version"), + qualifiers=purl_to_dict.get("qualifiers"), + subpath=purl_to_dict.get("subpath"), + ) + + assert type(vulnerablecode_package) == models.Package + assert ( + vulnerablecode_package.purl + == "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + ) + assert vulnerablecode_package.qualifiers == {} + assert vulnerablecode_package.subpath == "" From f920deda4961dbbbc5ecce16fcbecfff97618bcb Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 14 Aug 2023 17:42:05 -0700 Subject: [PATCH 07/53] Move weakness test #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/tests/test_models.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index fbd0803ff..58ee7cd79 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -61,6 +61,12 @@ def test_vulnerability_save_without_vulnerability_id(self): == 1 ) + def test_cwe_not_present_in_weaknesses_db(self): + w1 = models.Weakness.objects.create(name="189") + assert w1.weakness is None + assert w1.name is "" + assert w1.description is "" + # FIXME: The fixture code is duplicated. setUpClass is not working with the pytest mark. @pytest.mark.django_db @@ -190,13 +196,6 @@ def test_get_vulnerable_packages(self): assert vuln_packages.count() == 3 assert vuln_packages.distinct().count() == 2 - # matching_fixed_packages = vulnerablecode_package.get_fixed_packages(vulnerablecode_package) - # assert vuln_packages.distinct()[0] - - # matching_fixed_packages = vuln_packages.distinct()[0].get_fixed_packages( - # vuln_packages.distinct()[0] - # ) - first_vulnerable_package = vuln_packages.distinct()[0] matching_fixed_packages = first_vulnerable_package.get_fixed_packages( first_vulnerable_package @@ -400,9 +399,3 @@ def test_string_to_purl_to_dict_to_package(self): ) assert vulnerablecode_package.qualifiers == {} assert vulnerablecode_package.subpath == "" - - def test_cwe_not_present_in_weaknesses_db(self): - w1 = models.Weakness.objects.create(name="189") - assert w1.weakness is None - assert w1.name is "" - assert w1.description is "" From 40c1758324fff84943d55593098785428bd45681 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 14 Aug 2023 20:47:05 -0700 Subject: [PATCH 08/53] Modify UI, update dictionary and tests #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 15 ++++++++ .../templates/package_details.html | 35 ++++++++++++------- vulnerabilities/tests/test_models.py | 4 +++ vulnerablecode/static/css/custom.css | 2 +- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index cfa6e9d8a..1c728c15b 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -709,8 +709,11 @@ def fixed_package_details(self): purl_dict.update({"vulnerabilities": []}) purl_dict["closest_non_vulnerable_fix"] = "" + purl_dict["closest_non_vulnerable_fix_version"] = "" purl_dict["closest_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix"] = "" + purl_dict["most_recent_non_vulnerable_fix_version"] = "" purl_dict["most_recent_non_vulnerable_fix_url"] = "" for vuln in qs: @@ -761,7 +764,9 @@ def fixed_package_details(self): if dict_vuln["vulnerability"] == str(vuln): dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) + if len(vuln_matching_fixed_packages) > 0: + dict_vuln["closest_fixed_by_version"] = str(closest_fixed_package.version) dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() closest_fixed_package_vulns_dict = [ { @@ -774,19 +779,29 @@ def fixed_package_details(self): "closest_fixed_by_vulnerabilities" ] = closest_fixed_package_vulns_dict else: + dict_vuln["closest_fixed_by_version"] = "" dict_vuln["closest_fixed_by_url"] = "" dict_vuln["closest_fixed_by_vulnerabilities"] = [] purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + if len(vuln_matching_fixed_packages) > 0: + purl_dict["closest_non_vulnerable_fix_version"] = str( + closest_non_vulnerable_fix.version + ) purl_dict[ "closest_non_vulnerable_fix_url" ] = closest_non_vulnerable_fix.get_absolute_url() + purl_dict["most_recent_non_vulnerable_fix_version"] = str( + most_recent_non_vulnerable_fix.version + ) purl_dict[ "most_recent_non_vulnerable_fix_url" ] = most_recent_non_vulnerable_fix.get_absolute_url() else: + purl_dict["closest_non_vulnerable_fix_version"] = "" purl_dict["closest_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix_version"] = "" purl_dict["most_recent_non_vulnerable_fix_url"] = "" purl_dict["most_recent_non_vulnerable_fix"] = str( diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 236379ab6..1456945e3 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -47,18 +47,18 @@ - Closest non-vulnerable purl + Non-vulnerable version - {{ fixed_package_details.closest_non_vulnerable_fix }} + {{ fixed_package_details.closest_non_vulnerable_fix_version }} - Latest non-vulnerable purl + Latest non-vulnerable version - {{ fixed_package_details.most_recent_non_vulnerable_fix }} + {{ fixed_package_details.most_recent_non_vulnerable_fix_version }} @@ -80,7 +80,7 @@ Vulnerability Summary Aliases - Closest fixed by packages + Fixed by packages @@ -111,10 +111,24 @@ {% for abc in value %} {% if abc.vulnerability == vulnerability.vulnerability_id %} - {{ abc.closest_fixed_by_purl }} + {% if abc.closest_fixed_by_purl == "There are no reported fixed packages." %} + There are no reported fixed packages. + {% else %} + {{ abc.closest_fixed_by_purl }} + {% endif %}
- Vulnerabilities: {{ abc.closest_fixed_by_vulnerabilities|length }} + {% if abc.closest_fixed_by_purl == "There are no reported fixed packages." %} + + {% else %} + + {% if abc.closest_fixed_by_vulnerabilities|length != 1 %} + Affected by {{ abc.closest_fixed_by_vulnerabilities|length }} other vulnerabilities. + {% else %} + Affected by {{ abc.closest_fixed_by_vulnerabilities|length }} other vulnerability. + {% endif %} + + {% endif %} {% if abc.closest_fixed_by_vulnerabilities|length != 0 %}
@@ -125,12 +139,7 @@
- The closest fixed-by purl has {{ abc.closest_fixed_by_vulnerabilities|length }} - {% if abc.closest_fixed_by_vulnerabilities|length == 1 %} - vulnerability: - {% else %} - vulnerabilities: - {% endif %} + This version is affected by these other vulnerabilities:
{% for closest_vuln in abc.closest_fixed_by_vulnerabilities %}
diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 58ee7cd79..5ac0efeb1 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -218,6 +218,7 @@ def test_get_vulnerable_packages(self): { "vulnerability": "VCID-123", "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "closest_fixed_by_version": "2.13.2", "closest_fixed_by_url": "/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", "closest_fixed_by_vulnerabilities": [ { @@ -229,13 +230,16 @@ def test_get_vulnerable_packages(self): { "vulnerability": "VCID-456", "closest_fixed_by_purl": "There are no reported fixed packages.", + "closest_fixed_by_version": "", "closest_fixed_by_url": "", "closest_fixed_by_vulnerabilities": [], }, ], "closest_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "closest_non_vulnerable_fix_version": "", "closest_non_vulnerable_fix_url": "", "most_recent_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "most_recent_non_vulnerable_fix_version": "", "most_recent_non_vulnerable_fix_url": "", } diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index 38da36039..0f6d20abf 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -198,7 +198,7 @@ code { } .two-col-left { - width: 250px; + width: 255px; text-align: right !important; font-weight: bold; padding-right: 15px !important; From f3706713e6dae93e9aa027512a336e62d72b70b1 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 28 Aug 2023 09:31:27 -0700 Subject: [PATCH 09/53] Begin replacing strings with objects in package details dictionary #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 298 +++++++++++++----- .../templates/package_details.html | 227 +++++++++++-- vulnerabilities/tests/test_models.py | 32 +- vulnerablecode/settings.py | 1 + 4 files changed, 427 insertions(+), 131 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 1c728c15b..906f551d2 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -649,6 +649,7 @@ def get_sibling_packages(self, package): """ Return a queryset of all packages with the same type, namespace, name, subpath and qualifiers as the target package, whether or not the 'sibling packages' fix any vulnerability. + Identical to get_fixed_packages() above except for the removal of 'packagerelatedvulnerability__fix=True,'. """ return Package.objects.filter( name=package.name, @@ -659,16 +660,13 @@ def get_sibling_packages(self, package): ).distinct() def sort_by_version(self, packages_to_sort): - # Incoming is a list of + # Incoming is a list of objects. univers_version = RANGE_CLASS_BY_SCHEMES[packages_to_sort[0].type].version_class - sorted_by_version = [] - sorted_by_version = sorted( + return sorted( packages_to_sort, key=lambda x: univers_version(x.version), ) - return sorted_by_version - @property def fixed_package_details(self): """ @@ -678,7 +676,7 @@ def fixed_package_details(self): # Get all fixed packages that match the target package (type etc.), regardless of fixed vuln. matching_fixed_packages = self.get_fixed_packages(package=self) - # Get a list of the vulnerabilities that affect this package (i.e., self). + # Get a list of the vulnerabilities that affect the current package (i.e., self). qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) all_sibling_packages = self.get_sibling_packages(package=self) @@ -688,14 +686,17 @@ def fixed_package_details(self): if sib.is_vulnerable is False: non_vuln_sibs.append(sib) - # Add just the greater-than versions to a new list + # Add the greater-than versions to a new list. univers_version = RANGE_CLASS_BY_SCHEMES[self.type].version_class later_non_vuln_sibs = [] for non_vuln_sib in non_vuln_sibs: if univers_version(non_vuln_sib.version) > univers_version(self.version): later_non_vuln_sibs.append(non_vuln_sib) - # Take the list of vulns affecting the current package, retrieve a list of the fixed packages for each vuln, and assign the result to a custom attribute, 'matching_fixed_packages'. Ex: qs[0].matching_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). + # Take the list of vulnerabilities affecting the current package, retrieve a list of the fixed + # packages for each, and assign the result to a custom attribute, 'matching_fixed_packages'. + # Ex: qs[0].matching_fixed_packages returns the fixed package(s) for the 1st vulnerability + # for this affected package (i.e., self). qs = qs.prefetch_related( Prefetch( "packages", @@ -705,24 +706,34 @@ def fixed_package_details(self): ) purl_dict = {} - purl_dict["purl"] = self.purl - purl_dict.update({"vulnerabilities": []}) - - purl_dict["closest_non_vulnerable_fix"] = "" - purl_dict["closest_non_vulnerable_fix_version"] = "" - purl_dict["closest_non_vulnerable_fix_url"] = "" - - purl_dict["most_recent_non_vulnerable_fix"] = "" - purl_dict["most_recent_non_vulnerable_fix_version"] = "" - purl_dict["most_recent_non_vulnerable_fix_url"] = "" + # 2023-08-23 Wednesday 13:51:17. Let's remove this next. + # purl_dict["purl"] = self.purl + # Pass a PackageURL object to the Jinja2 template via the context dictionary. + purl_dict["test_purl"] = PackageURL.from_string(self.purl) + # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". + # purl_dict.update({"vulnerabilities": []}) + purl_dict.update({"test_vulnerabilities": []}) + + # # ID when there are 0 vulnerabilities. + # print("\nqs = {}\n".format(qs)) + # print("\nlen(qs) = {}\n".format(len(qs))) for vuln in qs: later_matching_fixed_packages = [] - purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". + # purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + # Pass a Vulnerability object. + purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) + # v1 = vuln + # print("\nv1 = {}\n".format(v1)) + # print("\ntype(v1) = {}\n".format(type(v1))) + # print("\nv1.severities = {}\n".format(v1.severities)) + # print("\nv1.affected_packages = {}\n".format(v1.affected_packages)) + vuln_matching_fixed_packages = vuln.matching_fixed_packages closest_fixed_package = "" - # Need to define here to avoid "" vs. [] for some closest_fixed_by_vulnerabilities values? - closest_fixed_package_vulns_dict = [] + # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. + # closest_fixed_package_vulns_dict = [] if len(vuln_matching_fixed_packages) > 0: for fixed_pkg in vuln_matching_fixed_packages: @@ -731,91 +742,216 @@ def fixed_package_details(self): ) > univers_version(self.version): later_matching_fixed_packages.append(fixed_pkg) - sort_fixed_by_packages_by_version = self.sort_by_version( - later_matching_fixed_packages - ) + # We already identified a closest fixed package and are looking for a later fixed package. + if later_matching_fixed_packages: + sort_fixed_by_packages_by_version = self.sort_by_version( + later_matching_fixed_packages + ) + # closest_fixed_package is a 'vulnerabilities.models.Package' closest_fixed_package = sort_fixed_by_packages_by_version[0] + closest_fixed_package_vulns = closest_fixed_package.affected_by - # 2023-08-13 Sunday 16:25:41. Not sure but I think I need to define this initially above just after closest_fixed_package = "" - closest_fixed_package_vulns_dict = [ - { - "vuln_id": fixed_pkg_vuln.vulnerability_id, - "vuln_get_absolute_url": fixed_pkg_vuln.get_absolute_url(), - } - for fixed_pkg_vuln in closest_fixed_package_vulns - ] + # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. + # closest_fixed_package_vulns_dict = [ + # { + # "vuln_id": fixed_pkg_vuln.vulnerability_id, + # } + # for fixed_pkg_vuln in closest_fixed_package_vulns + # ] else: closest_fixed_package = "There are no reported fixed packages." - for dict_vuln in purl_dict["vulnerabilities"]: - closest_non_vulnerable_fix = "" - - # TODO: 2023-08-13 Sunday 16:01:57. Refactor here and elsewhere! + # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". + # for dict_vuln in purl_dict["vulnerabilities"]: + # # for dict_vuln in purl_dict["test_vulnerabilities"]: + # closest_non_vulnerable_sib = "" + # if len(later_non_vuln_sibs) > 0: + # closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] + # else: + # closest_non_vulnerable_sib = "" + + # # most_recent_non_vulnerable_sib = "" + # latest_non_vulnerable_sib = "" + # if len(later_non_vuln_sibs) > 0: + # latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] + # else: + # latest_non_vulnerable_sib = "" + + # if dict_vuln["vulnerability"] == str(vuln): + # # ALERT: 2023-08-22 Tuesday 18:25:42. Try this. 2023-08-22 Tuesday 18:27:13. No does not work. + # # if dict_vuln["test_vulnerability"] == str(vuln): + + # if len(vuln_matching_fixed_packages) > 0: + # # dict_vuln["closest_fixed_by"] = purl_to_dict_with_purl( + # # closest_fixed_package.purl + # # ) + + # dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( + # closest_fixed_package.purl + # ) + # # print("For test_fixed_by_purl:") + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl) = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl) + # # ) + # # ) + # # print( + # # "\ntype(PackageURL.from_string(closest_fixed_package.purl)) = {}\n".format( + # # type(PackageURL.from_string(closest_fixed_package.purl)) + # # ) + # # ) + # # print( + # # "\nclosest_fixed_package.purl = {}\n".format(closest_fixed_package.purl) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).type = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).type + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).namespace = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).namespace + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).name = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).name + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).version = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).version + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).qualifiers = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).qualifiers + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).subpath = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).subpath + # # ) + # # ) + + # # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. + # # closest_fixed_package_vulns_dict = [ + # # { + # # "vuln_id": fixed_pkg_vuln.vulnerability_id, + # # } + # # for fixed_pkg_vuln in closest_fixed_package_vulns + # # ] + # # dict_vuln[ + # # "closest_fixed_by_vulnerabilities" + # # ] = closest_fixed_package_vulns_dict + + # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ + # fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns + # ] + # else: + # # dict_vuln["closest_fixed_by"] = {} + # # dict_vuln["closest_fixed_by_vulnerabilities"] = [] + + # dict_vuln["test_fixed_by_purl"] = None + # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] + + # if len(vuln_matching_fixed_packages) > 0: + # # purl_dict["closest_non_vulnerable"] = purl_to_dict_with_purl( + # # closest_non_vulnerable_sib.purl + # # ) + # # purl_dict["latest_non_vulnerable"] = purl_to_dict_with_purl( + # # latest_non_vulnerable_sib.purl + # # ) + + # purl_dict["test_closest_non_vulnerable"] = PackageURL.from_string( + # closest_non_vulnerable_sib.purl + # ) + # purl_dict["test_latest_non_vulnerable"] = PackageURL.from_string( + # latest_non_vulnerable_sib.purl + # ) + + # # purl_dict["most_recent_non_vulnerable_fix_version"] = str( + # # most_recent_non_vulnerable_fix.version + # # ) + # # purl_dict[ + # # "most_recent_non_vulnerable_fix_url" + # # ] = most_recent_non_vulnerable_fix.get_absolute_url() + # else: + # # purl_dict["closest_non_vulnerable"] = {} + # # purl_dict["latest_non_vulnerable"] = {} + + # # purl_dict["test_closest_non_vulnerable"] = "" + # # purl_dict["test_latest_non_vulnerable"] = "" + + # purl_dict["test_closest_non_vulnerable"] = None + # purl_dict["test_latest_non_vulnerable"] = None + + # ====================================================== + # TODO: 2023-08-22 Tuesday 18:37:15. Adapt this new section, from above, to the "test_vulnerabilities" approach, changing names as needed. + # for dict_vuln in purl_dict["vulnerabilities"]: + for dict_vuln in purl_dict["test_vulnerabilities"]: + closest_non_vulnerable_sib = "" if len(later_non_vuln_sibs) > 0: - closest_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[0] + closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] + else: + closest_non_vulnerable_sib = "" - most_recent_non_vulnerable_fix = "" + latest_non_vulnerable_sib = "" if len(later_non_vuln_sibs) > 0: - most_recent_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[-1] + latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] else: - most_recent_non_vulnerable_fix = "" + latest_non_vulnerable_sib = "" - if dict_vuln["vulnerability"] == str(vuln): - dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) + # if dict_vuln["vulnerability"] == str(vuln): + if dict_vuln["test_vulnerability"] == vuln: if len(vuln_matching_fixed_packages) > 0: - dict_vuln["closest_fixed_by_version"] = str(closest_fixed_package.version) - dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() - closest_fixed_package_vulns_dict = [ - { - "vuln_id": fixed_pkg_vuln.vulnerability_id, - "vuln_get_absolute_url": fixed_pkg_vuln.get_absolute_url(), - } - for fixed_pkg_vuln in closest_fixed_package_vulns + + dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( + closest_fixed_package.purl + ) + + # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. + # closest_fixed_package_vulns_dict = [ + # { + # "vuln_id": fixed_pkg_vuln.vulnerability_id, + # } + # for fixed_pkg_vuln in closest_fixed_package_vulns + # ] + + dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ + fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] - dict_vuln[ - "closest_fixed_by_vulnerabilities" - ] = closest_fixed_package_vulns_dict else: - dict_vuln["closest_fixed_by_version"] = "" - dict_vuln["closest_fixed_by_url"] = "" - dict_vuln["closest_fixed_by_vulnerabilities"] = [] - purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + dict_vuln["test_fixed_by_purl"] = None + dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] if len(vuln_matching_fixed_packages) > 0: - purl_dict["closest_non_vulnerable_fix_version"] = str( - closest_non_vulnerable_fix.version + + purl_dict["TEST_test_closest_non_vulnerable"] = PackageURL.from_string( + closest_non_vulnerable_sib.purl ) - purl_dict[ - "closest_non_vulnerable_fix_url" - ] = closest_non_vulnerable_fix.get_absolute_url() - purl_dict["most_recent_non_vulnerable_fix_version"] = str( - most_recent_non_vulnerable_fix.version + purl_dict["TEST_test_latest_non_vulnerable"] = PackageURL.from_string( + latest_non_vulnerable_sib.purl ) - purl_dict[ - "most_recent_non_vulnerable_fix_url" - ] = most_recent_non_vulnerable_fix.get_absolute_url() + else: - purl_dict["closest_non_vulnerable_fix_version"] = "" - purl_dict["closest_non_vulnerable_fix_url"] = "" - purl_dict["most_recent_non_vulnerable_fix_version"] = "" - purl_dict["most_recent_non_vulnerable_fix_url"] = "" - purl_dict["most_recent_non_vulnerable_fix"] = str( - most_recent_non_vulnerable_fix - ) + purl_dict["TEST_test_closest_non_vulnerable"] = None + purl_dict["TEST_test_latest_non_vulnerable"] = None + # ====================================================== - # Temporary print output during dev/testing: - # print("\npurl_dict = {}\n".format(purl_dict)) - print(json.dumps(purl_dict, indent=4, sort_keys=False)) + # Temporary print output during dev/testing. + from pprint import pprint - # TODO: Consider whether we want to provide the user with an option to output the dictionary to a file. - # pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) - # with open("pretty_purl_dict.txt", "w") as f: - # f.write(pretty_purl_dict) + pprint( + purl_dict, + sort_dicts=False, + ) + print("") return purl_dict diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 1456945e3..762935e95 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -33,7 +33,10 @@ - {{ package.purl }} + + {{ fixed_package_details.test_purl.to_string }} @@ -50,7 +53,24 @@ Non-vulnerable version - {{ fixed_package_details.closest_non_vulnerable_fix_version }} + Non-vulnerable version + + {{ fixed_package_details.TEST_test_closest_non_vulnerable.version }} + + + + + + + Non-vulnerable version
(non-breaking) + + + [to come] @@ -58,7 +78,22 @@ Latest non-vulnerable version - {{ fixed_package_details.most_recent_non_vulnerable_fix_version }} + Latest non-vulnerable version + + {{ fixed_package_details.TEST_test_latest_non_vulnerable.version }} + + + + + Latest non-vulnerable version
(non-breaking) + + + [to come] @@ -69,7 +104,7 @@
- Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }}) + Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }})
@@ -77,10 +112,9 @@ - + - - + @@ -89,11 +123,9 @@ - - + {% empty %} - @@ -175,13 +338,13 @@
- Fixed by vulnerabilities ({{ fixing_vulnerabilities|length }}) + Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }})
VulnerabilityVulnerability SummaryAliasesFixed by packagesFixed by
{{ vulnerability.vulnerability_id }} - - {{ vulnerability.summary }} - +
+ Aliases: +
{% for alias in vulnerability.alias %} {% if alias.url %} {{ alias }} @@ -104,36 +136,83 @@ {% endif %} {% endfor %} 		+ {{ vulnerability.summary }} + - {% if package.purl in fixed_package_details.purl %} + {# if package.purl in fixed_package_details.purl #} + {# if package.purl == fixed_package_details.test_purl.purl #} + {% if package.purl == fixed_package_details.test_purl.to_string %} + package.purl = {{ package.purl }} +
+ fixed_package_details.purl = {{ fixed_package_details.purl }} +
+ yikes +
+ fixed_package_details.test_purl = {{ fixed_package_details.test_purl }} +
+ fixed_package_details.test_purl.to_string = {{ fixed_package_details.test_purl.to_string }} +
+ {% for key, value in fixed_package_details.items %} - {% if key == "vulnerabilities" %} - {% for abc in value %} - {% if abc.vulnerability == vulnerability.vulnerability_id %} + {# if key == "vulnerabilities" #} + {% if key == "test_vulnerabilities" %} + goodby + {% for vuln in value %} +
+ vulnerability = {{ vulnerability }} +
+ vuln = {{ vuln }} +
+ vuln.test_vulnerability = {{ vuln.test_vulnerability }} +
+ vuln.test_vulnerability.to_string = {{ vuln.test_vulnerability.to_string }} +
+ vuln.test_vulnerability.vulnerability_id = {{ vuln.test_vulnerability.vulnerability_id }} +
+ vulnerability.vulnerability_id = {{ vulnerability.vulnerability_id }} + + {# if vuln.vulnerability == vulnerability.vulnerability_id #} + {% if vuln.test_vulnerability.vulnerability_id == vulnerability.vulnerability_id %} +
+ hello - {% if abc.closest_fixed_by_purl == "There are no reported fixed packages." %} - There are no reported fixed packages. + + + - {% if abc.closest_fixed_by_purl == "There are no reported fixed packages." %} + - {% if abc.closest_fixed_by_vulnerabilities|length != 0 %} + + + {% if vuln.test_fixed_by_purl is None %} + + + +
+ + There are no reported fixed versions. + +
+ + {% else %} + + {% if vuln.test_fixed_by_purl_vulnerabilities|length == 0 %} + + +
+ + {{ vuln.test_fixed_by_purl.version }} + +
+ Affected by 0 other vulnerabilities. + +
+ + {% else %} + + + +
+ + {{ vuln.test_fixed_by_purl.version }} + + {% if vuln.test_fixed_by_purl_vulnerabilities|length != 1 %} +
+ Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + {% else %} +
+ Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerability. + {% endif %} + +
+
+ +
+ +
+ +
+ + {% endif %} + {% endif %} + {% endif %} {% endfor %} {% endif %} @@ -164,7 +327,7 @@
+ This package is not known to be affected by vulnerabilities.
- + diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 5ac0efeb1..2bfa620af 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -217,30 +217,25 @@ def test_get_vulnerable_packages(self): "vulnerabilities": [ { "vulnerability": "VCID-123", - "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - "closest_fixed_by_version": "2.13.2", - "closest_fixed_by_url": "/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - "closest_fixed_by_vulnerabilities": [ - { - "vuln_id": "VCID-000", - "vuln_get_absolute_url": "/vulnerabilities/VCID-000", - } - ], + "closest_fixed_by": { + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "type": "maven", + "namespace": "com.fasterxml.jackson.core", + "name": "jackson-databind", + "version": "2.13.2", + "qualifiers": {}, + "subpath": "", + }, + "closest_fixed_by_vulnerabilities": [{"vuln_id": "VCID-000"}], }, { "vulnerability": "VCID-456", - "closest_fixed_by_purl": "There are no reported fixed packages.", - "closest_fixed_by_version": "", - "closest_fixed_by_url": "", + "closest_fixed_by": {}, "closest_fixed_by_vulnerabilities": [], }, ], - "closest_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "closest_non_vulnerable_fix_version": "", - "closest_non_vulnerable_fix_url": "", - "most_recent_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "most_recent_non_vulnerable_fix_version": "", - "most_recent_non_vulnerable_fix_url": "", + "closest_non_vulnerable": {}, + "latest_non_vulnerable": {}, } assert vuln_packages.distinct()[0].fixed_package_details == purl_dict @@ -337,6 +332,7 @@ def test_string_to_purl_to_dict_to_package(self): assert purl.subpath == None # Convert the PURL to a dictionary. + # ALERT: 2023-08-15 Tuesday 13:18:09. What about using the function 'def purl_to_dict(purl: PackageURL)'? Confusingly similar name but it seems designed to address the issue raised here (and looks useful for passing the data to the Jinja2 template). # It appears that this step is where the unwanted None values are created for qualifiers and # subpath when the PURL does not already contain values for those attributes. purl_to_dict = purl.to_dict() diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 3187b67ec..a632b710d 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -40,6 +40,7 @@ # SECURITY WARNING: do not run with debug turned on in production DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) +DEBUG = True # SECURITY WARNING: do not run with debug turned on in production DEBUG_TOOLBAR = env.bool("VULNERABLECODE_DEBUG_TOOLBAR", default=False) From 9ccaed76c566823105955356a98bc347b8e8f86f Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 28 Aug 2023 17:04:14 -0700 Subject: [PATCH 10/53] Clean current package details template and related model code #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 163 ------------ .../templates/package_details.html | 247 ++++-------------- 2 files changed, 46 insertions(+), 364 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 906f551d2..24c4da507 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -706,34 +706,17 @@ def fixed_package_details(self): ) purl_dict = {} - # 2023-08-23 Wednesday 13:51:17. Let's remove this next. - # purl_dict["purl"] = self.purl # Pass a PackageURL object to the Jinja2 template via the context dictionary. purl_dict["test_purl"] = PackageURL.from_string(self.purl) - # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". - # purl_dict.update({"vulnerabilities": []}) purl_dict.update({"test_vulnerabilities": []}) - # # ID when there are 0 vulnerabilities. - # print("\nqs = {}\n".format(qs)) - # print("\nlen(qs) = {}\n".format(len(qs))) - for vuln in qs: later_matching_fixed_packages = [] - # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". - # purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) # Pass a Vulnerability object. purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) - # v1 = vuln - # print("\nv1 = {}\n".format(v1)) - # print("\ntype(v1) = {}\n".format(type(v1))) - # print("\nv1.severities = {}\n".format(v1.severities)) - # print("\nv1.affected_packages = {}\n".format(v1.affected_packages)) vuln_matching_fixed_packages = vuln.matching_fixed_packages closest_fixed_package = "" - # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. - # closest_fixed_package_vulns_dict = [] if len(vuln_matching_fixed_packages) > 0: for fixed_pkg in vuln_matching_fixed_packages: @@ -748,149 +731,13 @@ def fixed_package_details(self): later_matching_fixed_packages ) - # closest_fixed_package is a 'vulnerabilities.models.Package' closest_fixed_package = sort_fixed_by_packages_by_version[0] closest_fixed_package_vulns = closest_fixed_package.affected_by - # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. - # closest_fixed_package_vulns_dict = [ - # { - # "vuln_id": fixed_pkg_vuln.vulnerability_id, - # } - # for fixed_pkg_vuln in closest_fixed_package_vulns - # ] else: closest_fixed_package = "There are no reported fixed packages." - # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". - # for dict_vuln in purl_dict["vulnerabilities"]: - # # for dict_vuln in purl_dict["test_vulnerabilities"]: - # closest_non_vulnerable_sib = "" - # if len(later_non_vuln_sibs) > 0: - # closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - # else: - # closest_non_vulnerable_sib = "" - - # # most_recent_non_vulnerable_sib = "" - # latest_non_vulnerable_sib = "" - # if len(later_non_vuln_sibs) > 0: - # latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] - # else: - # latest_non_vulnerable_sib = "" - - # if dict_vuln["vulnerability"] == str(vuln): - # # ALERT: 2023-08-22 Tuesday 18:25:42. Try this. 2023-08-22 Tuesday 18:27:13. No does not work. - # # if dict_vuln["test_vulnerability"] == str(vuln): - - # if len(vuln_matching_fixed_packages) > 0: - # # dict_vuln["closest_fixed_by"] = purl_to_dict_with_purl( - # # closest_fixed_package.purl - # # ) - - # dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( - # closest_fixed_package.purl - # ) - # # print("For test_fixed_by_purl:") - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl) = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl) - # # ) - # # ) - # # print( - # # "\ntype(PackageURL.from_string(closest_fixed_package.purl)) = {}\n".format( - # # type(PackageURL.from_string(closest_fixed_package.purl)) - # # ) - # # ) - # # print( - # # "\nclosest_fixed_package.purl = {}\n".format(closest_fixed_package.purl) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).type = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).type - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).namespace = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).namespace - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).name = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).name - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).version = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).version - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).qualifiers = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).qualifiers - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).subpath = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).subpath - # # ) - # # ) - - # # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. - # # closest_fixed_package_vulns_dict = [ - # # { - # # "vuln_id": fixed_pkg_vuln.vulnerability_id, - # # } - # # for fixed_pkg_vuln in closest_fixed_package_vulns - # # ] - # # dict_vuln[ - # # "closest_fixed_by_vulnerabilities" - # # ] = closest_fixed_package_vulns_dict - - # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ - # fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns - # ] - # else: - # # dict_vuln["closest_fixed_by"] = {} - # # dict_vuln["closest_fixed_by_vulnerabilities"] = [] - - # dict_vuln["test_fixed_by_purl"] = None - # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] - - # if len(vuln_matching_fixed_packages) > 0: - # # purl_dict["closest_non_vulnerable"] = purl_to_dict_with_purl( - # # closest_non_vulnerable_sib.purl - # # ) - # # purl_dict["latest_non_vulnerable"] = purl_to_dict_with_purl( - # # latest_non_vulnerable_sib.purl - # # ) - - # purl_dict["test_closest_non_vulnerable"] = PackageURL.from_string( - # closest_non_vulnerable_sib.purl - # ) - # purl_dict["test_latest_non_vulnerable"] = PackageURL.from_string( - # latest_non_vulnerable_sib.purl - # ) - - # # purl_dict["most_recent_non_vulnerable_fix_version"] = str( - # # most_recent_non_vulnerable_fix.version - # # ) - # # purl_dict[ - # # "most_recent_non_vulnerable_fix_url" - # # ] = most_recent_non_vulnerable_fix.get_absolute_url() - # else: - # # purl_dict["closest_non_vulnerable"] = {} - # # purl_dict["latest_non_vulnerable"] = {} - - # # purl_dict["test_closest_non_vulnerable"] = "" - # # purl_dict["test_latest_non_vulnerable"] = "" - - # purl_dict["test_closest_non_vulnerable"] = None - # purl_dict["test_latest_non_vulnerable"] = None - - # ====================================================== - # TODO: 2023-08-22 Tuesday 18:37:15. Adapt this new section, from above, to the "test_vulnerabilities" approach, changing names as needed. - # for dict_vuln in purl_dict["vulnerabilities"]: for dict_vuln in purl_dict["test_vulnerabilities"]: closest_non_vulnerable_sib = "" if len(later_non_vuln_sibs) > 0: @@ -904,7 +751,6 @@ def fixed_package_details(self): else: latest_non_vulnerable_sib = "" - # if dict_vuln["vulnerability"] == str(vuln): if dict_vuln["test_vulnerability"] == vuln: if len(vuln_matching_fixed_packages) > 0: @@ -913,14 +759,6 @@ def fixed_package_details(self): closest_fixed_package.purl ) - # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. - # closest_fixed_package_vulns_dict = [ - # { - # "vuln_id": fixed_pkg_vuln.vulnerability_id, - # } - # for fixed_pkg_vuln in closest_fixed_package_vulns - # ] - dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] @@ -942,7 +780,6 @@ def fixed_package_details(self): purl_dict["TEST_test_closest_non_vulnerable"] = None purl_dict["TEST_test_latest_non_vulnerable"] = None - # ====================================================== # Temporary print output during dev/testing. from pprint import pprint diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 762935e95..58e433171 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -33,9 +33,6 @@ @@ -45,60 +42,44 @@ {% if affected_by_vulnerabilities|length != 0 %} -
-
VulnerabilityVulnerability Summary Aliases
- {{ fixed_package_details.test_purl.to_string }}
- - - - - - - - - - - - - - - - - - -
- Non-vulnerable version - - Non-vulnerable version - - {{ fixed_package_details.TEST_test_closest_non_vulnerable.version }} - - -
- Non-vulnerable version
(non-breaking) - 		- [to come] -
- Latest non-vulnerable version - - Latest non-vulnerable version - - {{ fixed_package_details.TEST_test_latest_non_vulnerable.version }} -
- Latest non-vulnerable version
(non-breaking) - 		- [to come] -
-
+
+ + + + + + + + + + + + + + + + + + + +
+ Non-vulnerable version + + {{ fixed_package_details.TEST_test_closest_non_vulnerable.version }} +
+ Non-vulnerable version
(non-breaking) + 		+ [to come] +
+ Latest non-vulnerable version + + {{ fixed_package_details.TEST_test_latest_non_vulnerable.version }} +
+ Latest non-vulnerable version
(non-breaking) + 		+ [to come] +
+
{% endif %} @@ -107,8 +88,6 @@ Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }})
- - @@ -140,154 +119,26 @@ {{ vulnerability.summary }} @@ -385,5 +231,4 @@ {% endif %} - {% endblock %} From 1a26613522d3079c4dc271248938995c00cde1c3 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Wed, 30 Aug 2023 12:20:12 -0700 Subject: [PATCH 11/53] Begin work on major-version issue #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 128 +++++++++++++++--- .../templates/package_details.html | 38 +++--- vulnerabilities/tests/test_models.py | 110 +++++++++++++++ 3 files changed, 239 insertions(+), 37 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 24c4da507..af16f66eb 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -688,6 +688,7 @@ def fixed_package_details(self): # Add the greater-than versions to a new list. univers_version = RANGE_CLASS_BY_SCHEMES[self.type].version_class + print("\nunivers_version = {}".format(univers_version)) later_non_vuln_sibs = [] for non_vuln_sib in non_vuln_sibs: if univers_version(non_vuln_sib.version) > univers_version(self.version): @@ -707,13 +708,34 @@ def fixed_package_details(self): purl_dict = {} # Pass a PackageURL object to the Jinja2 template via the context dictionary. - purl_dict["test_purl"] = PackageURL.from_string(self.purl) - purl_dict.update({"test_vulnerabilities": []}) + # purl_dict["test_purl"] = PackageURL.from_string(self.purl) + # purl_dict.update({"test_vulnerabilities": []}) + purl_dict["purl"] = PackageURL.from_string(self.purl) + print("\npurl = {}".format(purl_dict["purl"])) + + purl_univers_version = univers_version(purl_dict["purl"].version) + print("\npurl_univers_version = {}".format(purl_univers_version)) + + # ALERT: This throws an error: AttributeError: 'MavenVersion' object has no attribute 'major' + # How do we compare major versions? + # purl_univers_version_major = purl_univers_version.major + + # purl_semver_version = versions.SemverVersion(purl_dict["purl"].version) + # print("\npurl_semver_version = {}".format(purl_semver_version)) + + # purl_semver_version_major = purl_semver_version.major + # print("\npurl_semver_version_major = {}".format(purl_semver_version_major)) + + purl_semver_version_major = versions.SemverVersion(purl_dict["purl"].version).major + print("\npurl_semver_version_major = {}\n".format(purl_semver_version_major)) + + purl_dict.update({"vulnerabilities": []}) for vuln in qs: later_matching_fixed_packages = [] # Pass a Vulnerability object. - purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) + # purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) + purl_dict["vulnerabilities"].append({"vulnerability": vuln}) vuln_matching_fixed_packages = vuln.matching_fixed_packages closest_fixed_package = "" @@ -730,6 +752,29 @@ def fixed_package_details(self): sort_fixed_by_packages_by_version = self.sort_by_version( later_matching_fixed_packages ) + print( + "\nsort_fixed_by_packages_by_version = {}\n".format( + sort_fixed_by_packages_by_version + ) + ) + for fixed_by_pkg in sort_fixed_by_packages_by_version: + print("\nfixed_by_pkg.version = {}\n".format(fixed_by_pkg.version)) + purl_from_fixed_by_pkg = PackageURL.from_string(fixed_by_pkg.purl) + print("\npurl_from_fixed_by_pkg = {}\n".format(purl_from_fixed_by_pkg)) + + print("\nfixed_by_pkg = {}\n".format(fixed_by_pkg)) + + # purl_semver_version_major = versions.SemverVersion(purl_dict["purl"].version).major + # print("\npurl_semver_version_major = {}\n".format(purl_semver_version_major)) + + fixed_by_pkg_semver_version_major = versions.SemverVersion( + fixed_by_pkg.version + ).major + print( + "\nfixed_by_pkg_semver_version_major = {}\n".format( + fixed_by_pkg_semver_version_major + ) + ) closest_fixed_package = sort_fixed_by_packages_by_version[0] @@ -738,48 +783,95 @@ def fixed_package_details(self): else: closest_fixed_package = "There are no reported fixed packages." - for dict_vuln in purl_dict["test_vulnerabilities"]: + # for dict_vuln in purl_dict["test_vulnerabilities"]: + for dict_vuln in purl_dict["vulnerabilities"]: closest_non_vulnerable_sib = "" - if len(later_non_vuln_sibs) > 0: - closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - else: - closest_non_vulnerable_sib = "" + # if len(later_non_vuln_sibs) > 0: + # closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] + # else: + # closest_non_vulnerable_sib = "" latest_non_vulnerable_sib = "" + # if len(later_non_vuln_sibs) > 0: + # latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] + # else: + # latest_non_vulnerable_sib = "" + + closest_non_vulnerable_sib_non_breaking = "" + latest_non_vulnerable_sib_non_breaking = "" + if len(later_non_vuln_sibs) > 0: + closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] + + print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) + + later_non_vuln_sibs_non_breaking = [] + for non_vuln_sib in later_non_vuln_sibs: + print("\nnon_vuln_sib = {}\n".format(non_vuln_sib)) + print("\ntype(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) + + # 2023-08-28 Monday 19:19:03. I need to resume here and populate the list later_non_vuln_sibs_non_breaking, then see whether there are any non-breaking versions and in anmy event add 2 more dict entries for the closest non-breaking and latest non-breaking versions. else: + closest_non_vulnerable_sib = "" latest_non_vulnerable_sib = "" + # FIXME: 2023-08-30 Wednesday 11:18:43. Got an error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'. I think the if condition below needs to be changed from if len(vuln_matching_fixed_packages) > 0: to if len(later_non_vuln_sibs) > 0:. Yes that fixes the error! + # TODO: With the fix above, we also need to consolidate this if with the if below -- both start with "if len(later_non_vuln_sibs) > 0:" - if dict_vuln["test_vulnerability"] == vuln: + # if dict_vuln["test_vulnerability"] == vuln: + if dict_vuln["vulnerability"] == vuln: if len(vuln_matching_fixed_packages) > 0: - dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( + # dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( + # closest_fixed_package.purl + # ) + + # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ + # fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns + # ] + + dict_vuln["fixed_by_purl"] = PackageURL.from_string( closest_fixed_package.purl ) - dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ + dict_vuln["fixed_by_purl_vulnerabilities"] = [ fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] else: - dict_vuln["test_fixed_by_purl"] = None - dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] + # dict_vuln["test_fixed_by_purl"] = None + # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] - if len(vuln_matching_fixed_packages) > 0: + dict_vuln["fixed_by_purl"] = None + dict_vuln["fixed_by_purl_vulnerabilities"] = [] - purl_dict["TEST_test_closest_non_vulnerable"] = PackageURL.from_string( + # FIXME: 2023-08-30 Wednesday 11:29:34. Will this handle the error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'? Yes! + # TODO: Having fixed this, we also need to conslidate this if with the similar if above. + # if len(vuln_matching_fixed_packages) > 0: + if len(later_non_vuln_sibs) > 0: + + # purl_dict["TEST_test_closest_non_vulnerable"] = PackageURL.from_string( + # closest_non_vulnerable_sib.purl + # ) + # purl_dict["TEST_test_latest_non_vulnerable"] = PackageURL.from_string( + # latest_non_vulnerable_sib.purl + # ) + + purl_dict["closest_non_vulnerable"] = PackageURL.from_string( closest_non_vulnerable_sib.purl ) - purl_dict["TEST_test_latest_non_vulnerable"] = PackageURL.from_string( + purl_dict["latest_non_vulnerable"] = PackageURL.from_string( latest_non_vulnerable_sib.purl ) else: - purl_dict["TEST_test_closest_non_vulnerable"] = None - purl_dict["TEST_test_latest_non_vulnerable"] = None + # purl_dict["TEST_test_closest_non_vulnerable"] = None + # purl_dict["TEST_test_latest_non_vulnerable"] = None + + purl_dict["closest_non_vulnerable"] = None + purl_dict["latest_non_vulnerable"] = None # Temporary print output during dev/testing. from pprint import pprint diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 58e433171..c1e1bff6f 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -33,7 +33,7 @@ @@ -50,33 +50,33 @@ Non-vulnerable version - + - +
- {# if package.purl in fixed_package_details.purl #} - {# if package.purl == fixed_package_details.test_purl.purl #} {% if package.purl == fixed_package_details.test_purl.to_string %} - package.purl = {{ package.purl }} -
- fixed_package_details.purl = {{ fixed_package_details.purl }} -
- yikes -
- fixed_package_details.test_purl = {{ fixed_package_details.test_purl }} -
- fixed_package_details.test_purl.to_string = {{ fixed_package_details.test_purl.to_string }} -
- {% for key, value in fixed_package_details.items %} - {# if key == "vulnerabilities" #} {% if key == "test_vulnerabilities" %} - goodby {% for vuln in value %} -
- vulnerability = {{ vulnerability }} -
- vuln = {{ vuln }} -
- vuln.test_vulnerability = {{ vuln.test_vulnerability }} -
- vuln.test_vulnerability.to_string = {{ vuln.test_vulnerability.to_string }} -
- vuln.test_vulnerability.vulnerability_id = {{ vuln.test_vulnerability.vulnerability_id }} -
- vulnerability.vulnerability_id = {{ vulnerability.vulnerability_id }} - - {# if vuln.vulnerability == vulnerability.vulnerability_id #} {% if vuln.test_vulnerability.vulnerability_id == vulnerability.vulnerability_id %} -
- hello - - - - - - - - - {% if vuln.test_fixed_by_purl is None %} - - - -
- There are no reported fixed versions. - -
- {% else %} - {% if vuln.test_fixed_by_purl_vulnerabilities|length == 0 %} - - -
- {{ vuln.test_fixed_by_purl.version }} -
Affected by 0 other vulnerabilities. - -
- {% else %} - - - -
- {{ vuln.test_fixed_by_purl.version }} - {% if vuln.test_fixed_by_purl_vulnerabilities|length != 1 %} -
- Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerabilities. - {% else %} -
- Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerability. +
+ Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + {% else %} +
+ Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerability. {% endif %}
@@ -301,9 +152,9 @@ This version is affected by these other vulnerabilities:
{% for fixed_by_vuln in vuln.test_fixed_by_purl_vulnerabilities %} - + {% endfor %}
@@ -311,17 +162,12 @@ - -
- {% endif %} {% endif %} - {% endif %} {% endfor %} {% endif %} {% endfor %} - {% endif %}
- {{ fixed_package_details.test_purl.to_string }} + {{ fixed_package_details.purl.to_string }}
- {{ fixed_package_details.TEST_test_closest_non_vulnerable.version }} + {{ fixed_package_details.closest_non_vulnerable.version }}
Latest non-vulnerable version - {{ fixed_package_details.TEST_test_latest_non_vulnerable.version }} + {{ fixed_package_details.latest_non_vulnerable.version }}
@@ -119,26 +119,26 @@ {{ vulnerability.summary }} - {% if package.purl == fixed_package_details.test_purl.to_string %} + {% if package.purl == fixed_package_details.purl.to_string %} {% for key, value in fixed_package_details.items %} - {% if key == "test_vulnerabilities" %} + {% if key == "vulnerabilities" %} {% for vuln in value %} - {% if vuln.test_vulnerability.vulnerability_id == vulnerability.vulnerability_id %} - {% if vuln.test_fixed_by_purl is None %} + {% if vuln.vulnerability.vulnerability_id == vulnerability.vulnerability_id %} + {% if vuln.fixed_by_purl is None %} There are no reported fixed versions. {% else %} - {% if vuln.test_fixed_by_purl_vulnerabilities|length == 0 %} - {{ vuln.test_fixed_by_purl.version }} + {% if vuln.fixed_by_purl_vulnerabilities|length == 0 %} + {{ vuln.fixed_by_purl.version }}
- Affected by 0 other vulnerabilities. + Affected by 0 other vulnerabilities. {% else %} - {{ vuln.test_fixed_by_purl.version }} - {% if vuln.test_fixed_by_purl_vulnerabilities|length != 1 %} + {{ vuln.fixed_by_purl.version }} + {% if vuln.fixed_by_purl_vulnerabilities|length != 1 %}
- Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. {% else %}
- Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerability. + Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerability. {% endif %}
@@ -151,7 +151,7 @@
This version is affected by these other vulnerabilities:
- {% for fixed_by_vuln in vuln.test_fixed_by_purl_vulnerabilities %} + {% for fixed_by_vuln in vuln.fixed_by_purl_vulnerabilities %} diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 2bfa620af..d6b17dfb1 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -191,6 +191,87 @@ def setUp(self): fix=False, ) + # Create additional Package objects with various versions to test the major version identification and comparison process. + + self.pypi_setuptools_affected_package = models.Package.objects.create( + type="pypi", + namespace="", + name="setuptools", + version="40.8.0", + qualifiers={}, + subpath="", + ) + + self.pypi_setuptools_fixed_closest_and_latest_non_vulnerable_packages = ( + models.Package.objects.create( + type="pypi", + namespace="", + name="setuptools", + version="65.5.1", + qualifiers={}, + subpath="", + ) + ) + + # pkg:maven/org.eclipse.jetty/jetty-util@9.3.20.v20170531 + + # from the string + jetty_util_purl = PackageURL.from_string( + "pkg:maven/org.eclipse.jetty/jetty-util@9.3.20.v20170531" + ) + + # # convert purl to dict + # jetty_util_purl_to_dict = jetty_util_purl.to_dict() + # # This will avoid the IntegrityError: + # if jetty_util_purl_to_dict.get("qualifiers") is None: + # jetty_util_purl_to_dict["qualifiers"] = {} + # if jetty_util_purl_to_dict.get("subpath") is None: + # jetty_util_purl_to_dict["subpath"] = "" + + # This takes the place of the 2 preceding bits -- uses purl_to_dict() rather than just to_dict() + jetty_util_purl_to_dict = models.purl_to_dict(jetty_util_purl) + + # convert dict to Package + # This needs self, right? + self.maven_jetty_util_affected_package = models.Package.objects.create( + type=jetty_util_purl_to_dict.get("type"), + namespace=jetty_util_purl_to_dict.get("namespace"), + name=jetty_util_purl_to_dict.get("name"), + version=jetty_util_purl_to_dict.get("version"), + qualifiers=jetty_util_purl_to_dict.get("qualifiers"), + subpath=jetty_util_purl_to_dict.get("subpath"), + ) + + # using the create method + # self.maven_jetty_util_affected_package = models.Package.objects.create( + # type="maven", + # namespace="org.eclipse.jetty", + # name="jetty-util", + # version="9.3.20.v20170531", + # qualifiers={}, + # subpath="", + # ) + # pkg:maven/org.eclipse.jetty/jetty-util@9.4.39.v20210325 + self.maven_jetty_util_fixed_package = models.Package.objects.create( + type="maven", + namespace="org.eclipse.jetty", + name="jetty-util", + version="9.4.39.v20210325", + qualifiers={}, + subpath="", + ) + # pkg:maven/org.eclipse.jetty/jetty-util@11.0.14 + self.maven_jetty_util_closest_and_latest_non_vulnerable_packages = ( + models.Package.objects.create( + type="maven", + namespace="org.eclipse.jetty", + name="jetty-util", + version="11.0.14", + qualifiers={}, + subpath="", + ) + ) + def test_get_vulnerable_packages(self): vuln_packages = Package.objects.vulnerable() assert vuln_packages.count() == 3 @@ -399,3 +480,32 @@ def test_string_to_purl_to_dict_to_package(self): ) assert vulnerablecode_package.qualifiers == {} assert vulnerablecode_package.subpath == "" + + def test_compare_package_major_versions(self): + # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., + # a . + + # Convert a PURL string to a PURL. + purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + purl = PackageURL.from_string(purl_string) + + assert type(purl) == PackageURL + assert purl.type == "maven" + assert purl.qualifiers == {} + assert purl.subpath == None + + print("\npurl_string = {}".format(purl_string)) + + print("\npurl = {}".format(purl)) + + print("\nHello VulnerableCode!\n") + + all_packages = Package.objects + print("\nPackage.objects = {}\n".format(Package.objects)) + print("\nall_packages.distinct() = {}\n".format(all_packages.distinct())) + print("\nall_packages.distinct()[0] = {}\n".format(all_packages.distinct()[0])) + + for pkg in all_packages.distinct(): + print(PackageURL.from_string(pkg.purl)) + + print("") From b6267622542fd18cc0b0d6d8dc1bfabf4a7ca05a Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 5 Sep 2023 19:03:35 -0700 Subject: [PATCH 12/53] Complete first round of major-version vetting #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 295 ++++++++++-------- .../templates/package_details.html | 23 +- 2 files changed, 186 insertions(+), 132 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index af16f66eb..e2a196eb4 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -686,18 +686,73 @@ def fixed_package_details(self): if sib.is_vulnerable is False: non_vuln_sibs.append(sib) - # Add the greater-than versions to a new list. - univers_version = RANGE_CLASS_BY_SCHEMES[self.type].version_class - print("\nunivers_version = {}".format(univers_version)) + # Get the univers version_class for the purl, i.e., the Package the user has searched for. + univers_version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class + # print("\nunivers_version_class = {}".format(univers_version_class)) + + # ======================================================= + # Get the univers version for the purl. + purl_univers_version = univers_version_class(PackageURL.from_string(self.purl).version) + # print("\n>>> purl_univers_version = {}\n".format(purl_univers_version)) + # 2023-09-04 Monday 18:23:51. Is the version just a string I can split? + # print("\n>>> type(purl_univers_version) = {}\n".format(type(purl_univers_version))) + # print( + # "\n>>> type(purl_univers_version.__str__()) = {}\n".format( + # type(purl_univers_version.__str__()) + # ) + # ) + # print( + # "\n>>> purl_univers_version.__str__().split('.')[0] = {}\n".format( + # purl_univers_version.__str__().split(".")[0] + # ) + # ) + + # Convert the univers version for the purl to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. + purl_univers_version_major_int = int(purl_univers_version.__str__().split(".")[0]) + # print("\n>>> purl_univers_version_major_int = {}\n".format(purl_univers_version_major_int)) + # print( + # "\n>>> type(purl_univers_version_major_int) = {}\n".format( + # type(purl_univers_version_major_int) + # ) + # ) + + # Create a list for the later non-vuln siblings. later_non_vuln_sibs = [] + # Each non_vuln_sib is a Package. for non_vuln_sib in non_vuln_sibs: - if univers_version(non_vuln_sib.version) > univers_version(self.version): + + # print("\n### type(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) + + # Get the univers version for the non_vuln_sib. + non_vuln_sib_univers_version = univers_version_class( + PackageURL.from_string(non_vuln_sib.purl).version + ) + + # print("\n### non_vuln_sib_univers_version = {}\n".format(non_vuln_sib_univers_version)) + + # Convert the univers version for the purl to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. + non_vuln_sib_univers_version_major_int = int( + non_vuln_sib_univers_version.__str__().split(".")[0] + ) + + # print( + # "\n### non_vuln_sib_univers_version_major_int = {}\n".format( + # non_vuln_sib_univers_version_major_int + # ) + # ) + + # Add it to the 'later_non_vuln_sibs' list if the major versions are the same. + # if univers_version_class(non_vuln_sib.version) > univers_version_class(self.version): + if ( + univers_version_class(non_vuln_sib.version) > univers_version_class(self.version) + and purl_univers_version_major_int == non_vuln_sib_univers_version_major_int + ): later_non_vuln_sibs.append(non_vuln_sib) # Take the list of vulnerabilities affecting the current package, retrieve a list of the fixed - # packages for each, and assign the result to a custom attribute, 'matching_fixed_packages'. - # Ex: qs[0].matching_fixed_packages returns the fixed package(s) for the 1st vulnerability - # for this affected package (i.e., self). + # packages for each (self.get_fixed_packages()), and assign the result to a custom attribute, 'matching_fixed_packages'. + # Ex: qs[0].matching_fixed_packages returns the fixed package(s) for the 1st vulnerability. + qs = qs.prefetch_related( Prefetch( "packages", @@ -708,43 +763,101 @@ def fixed_package_details(self): purl_dict = {} # Pass a PackageURL object to the Jinja2 template via the context dictionary. - # purl_dict["test_purl"] = PackageURL.from_string(self.purl) - # purl_dict.update({"test_vulnerabilities": []}) purl_dict["purl"] = PackageURL.from_string(self.purl) - print("\npurl = {}".format(purl_dict["purl"])) + # print("\npurl = {}".format(purl_dict["purl"])) - purl_univers_version = univers_version(purl_dict["purl"].version) - print("\npurl_univers_version = {}".format(purl_univers_version)) + closest_non_vulnerable_sib = "" + latest_non_vulnerable_sib = "" - # ALERT: This throws an error: AttributeError: 'MavenVersion' object has no attribute 'major' - # How do we compare major versions? - # purl_univers_version_major = purl_univers_version.major + # closest_non_vulnerable_sib_non_breaking = "" + # latest_non_vulnerable_sib_non_breaking = "" - # purl_semver_version = versions.SemverVersion(purl_dict["purl"].version) - # print("\npurl_semver_version = {}".format(purl_semver_version)) + # We've already filled later_non_vuln_sibs with all non_vuln sibs with the same major + # version as the purl the user is searching for (i.e., self) -- but this could be an empty list. + if len(later_non_vuln_sibs) > 0: + closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] + latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] - # purl_semver_version_major = purl_semver_version.major - # print("\npurl_semver_version_major = {}".format(purl_semver_version_major)) + purl_dict["closest_non_vulnerable"] = PackageURL.from_string( + closest_non_vulnerable_sib.purl + ) + purl_dict["latest_non_vulnerable"] = PackageURL.from_string( + latest_non_vulnerable_sib.purl + ) - purl_semver_version_major = versions.SemverVersion(purl_dict["purl"].version).major - print("\npurl_semver_version_major = {}\n".format(purl_semver_version_major)) + # print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) + + # later_non_vuln_sibs_non_breaking = [] + # for non_vuln_sib in later_non_vuln_sibs: + # print("\nnon_vuln_sib = {}\n".format(non_vuln_sib)) + # print("\ntype(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) + + else: + # closest_non_vulnerable_sib = "" + # latest_non_vulnerable_sib = "" + + purl_dict["closest_non_vulnerable"] = None + purl_dict["latest_non_vulnerable"] = None + # ======================================================== + # if len(later_non_vuln_sibs) > 0: + # purl_dict["closest_non_vulnerable"] = PackageURL.from_string( + # closest_non_vulnerable_sib.purl + # ) + # purl_dict["latest_non_vulnerable"] = PackageURL.from_string( + # latest_non_vulnerable_sib.purl + # ) + # else: + # purl_dict["closest_non_vulnerable"] = None + # purl_dict["latest_non_vulnerable"] = None + # ======================================================== + + # No longer need this. + # purl_univers_version_test = univers_version_class(purl_dict["purl"].version) + # print("\npurl_univers_version_test = {}".format(purl_univers_version_test)) purl_dict.update({"vulnerabilities": []}) + # For each vulnerability affecting the current Package... for vuln in qs: later_matching_fixed_packages = [] # Pass a Vulnerability object. - # purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) purl_dict["vulnerabilities"].append({"vulnerability": vuln}) + # Using the qs prefetch, get a of all the matching fixed packages for this vulnerability, all major versions to start with. vuln_matching_fixed_packages = vuln.matching_fixed_packages + # print( + # "\n??? type(vuln_matching_fixed_packages) = {}\n".format( + # type(vuln_matching_fixed_packages) + # ) + # ) closest_fixed_package = "" + # ALERT: 2023-09-05 Tuesday 17:42:58. When refreshing a Package details page, I got an error: local variable 'sort_fixed_by_packages_by_version' referenced before assignment (referring to the definition of closest_fixed_package below) + # ALERT: Do I need to define it here? + sort_fixed_by_packages_by_version = [] + if len(vuln_matching_fixed_packages) > 0: for fixed_pkg in vuln_matching_fixed_packages: - if fixed_pkg in matching_fixed_packages and univers_version( - fixed_pkg.version - ) > univers_version(self.version): + # fixed_pkg is a 'vulnerabilities.models.Package' + # print("\n!!! type(fixed_pkg) = {}\n".format(type(fixed_pkg))) + + # Get the univers version for the fixed_pkg. + fixed_pkg_univers_version = univers_version_class( + PackageURL.from_string(fixed_pkg.purl).version + ) + + # Convert the univers version for the fixed_pkg to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. + fixed_pkg_univers_version_major_int = int( + fixed_pkg_univers_version.__str__().split(".")[0] + ) + + # Add it to the 'later_non_vuln_sibs' list if the major versions are the same. + if ( + fixed_pkg in matching_fixed_packages + and univers_version_class(fixed_pkg.version) + > univers_version_class(self.version) + and purl_univers_version_major_int == fixed_pkg_univers_version_major_int + ): later_matching_fixed_packages.append(fixed_pkg) # We already identified a closest fixed package and are looking for a later fixed package. @@ -752,126 +865,56 @@ def fixed_package_details(self): sort_fixed_by_packages_by_version = self.sort_by_version( later_matching_fixed_packages ) - print( - "\nsort_fixed_by_packages_by_version = {}\n".format( - sort_fixed_by_packages_by_version - ) - ) - for fixed_by_pkg in sort_fixed_by_packages_by_version: - print("\nfixed_by_pkg.version = {}\n".format(fixed_by_pkg.version)) - purl_from_fixed_by_pkg = PackageURL.from_string(fixed_by_pkg.purl) - print("\npurl_from_fixed_by_pkg = {}\n".format(purl_from_fixed_by_pkg)) - - print("\nfixed_by_pkg = {}\n".format(fixed_by_pkg)) - - # purl_semver_version_major = versions.SemverVersion(purl_dict["purl"].version).major - # print("\npurl_semver_version_major = {}\n".format(purl_semver_version_major)) - - fixed_by_pkg_semver_version_major = versions.SemverVersion( - fixed_by_pkg.version - ).major - print( - "\nfixed_by_pkg_semver_version_major = {}\n".format( - fixed_by_pkg_semver_version_major - ) - ) + # print( + # "\nsort_fixed_by_packages_by_version = {}\n".format( + # sort_fixed_by_packages_by_version + # ) + # ) - closest_fixed_package = sort_fixed_by_packages_by_version[0] + # ALERT: 2023-09-05 Tuesday 17:53:54. I think these 2 lines need to be indented to be inside the if condition above. + closest_fixed_package = sort_fixed_by_packages_by_version[0] + closest_fixed_package_vulns = closest_fixed_package.affected_by - closest_fixed_package_vulns = closest_fixed_package.affected_by + # closest_fixed_package = sort_fixed_by_packages_by_version[0] + # closest_fixed_package_vulns = closest_fixed_package.affected_by else: - closest_fixed_package = "There are no reported fixed packages." + # ALERT: 2023-09-05 Tuesday 18:03:46. I don't think we want this string but rather something like None, for the if condition below. + # closest_fixed_package = "There are no reported fixed packages." + closest_fixed_package = None - # for dict_vuln in purl_dict["test_vulnerabilities"]: for dict_vuln in purl_dict["vulnerabilities"]: - closest_non_vulnerable_sib = "" - # if len(later_non_vuln_sibs) > 0: - # closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - # else: - # closest_non_vulnerable_sib = "" - - latest_non_vulnerable_sib = "" - # if len(later_non_vuln_sibs) > 0: - # latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] - # else: - # latest_non_vulnerable_sib = "" - - closest_non_vulnerable_sib_non_breaking = "" - latest_non_vulnerable_sib_non_breaking = "" - - if len(later_non_vuln_sibs) > 0: - closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] - - print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) - - later_non_vuln_sibs_non_breaking = [] - for non_vuln_sib in later_non_vuln_sibs: - print("\nnon_vuln_sib = {}\n".format(non_vuln_sib)) - print("\ntype(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) - - # 2023-08-28 Monday 19:19:03. I need to resume here and populate the list later_non_vuln_sibs_non_breaking, then see whether there are any non-breaking versions and in anmy event add 2 more dict entries for the closest non-breaking and latest non-breaking versions. - else: - closest_non_vulnerable_sib = "" - latest_non_vulnerable_sib = "" - # FIXME: 2023-08-30 Wednesday 11:18:43. Got an error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'. I think the if condition below needs to be changed from if len(vuln_matching_fixed_packages) > 0: to if len(later_non_vuln_sibs) > 0:. Yes that fixes the error! - # TODO: With the fix above, we also need to consolidate this if with the if below -- both start with "if len(later_non_vuln_sibs) > 0:" - - # if dict_vuln["test_vulnerability"] == vuln: if dict_vuln["vulnerability"] == vuln: - if len(vuln_matching_fixed_packages) > 0: - - # dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( - # closest_fixed_package.purl - # ) - - # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ - # fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns - # ] - + # ALERT: 2023-09-05 Tuesday 17:58:24. This is not the correct test because vuln_matching_fixed_packages might contain no major version identical to the purl major version! Instead we want to check whether the closest_fixed_package exists, which we defined above as closest_fixed_package = sort_fixed_by_packages_by_version[0] + # if len(vuln_matching_fixed_packages) > 0: + if closest_fixed_package: dict_vuln["fixed_by_purl"] = PackageURL.from_string( closest_fixed_package.purl ) + # ALERT: 2023-09-05 Tuesday 18:00:09. I expect we'll also see an error here and will need to define closest_fixed_package_vulns higher above than where it is currently defined. dict_vuln["fixed_by_purl_vulnerabilities"] = [ fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] else: - - # dict_vuln["test_fixed_by_purl"] = None - # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] - dict_vuln["fixed_by_purl"] = None dict_vuln["fixed_by_purl_vulnerabilities"] = [] - # FIXME: 2023-08-30 Wednesday 11:29:34. Will this handle the error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'? Yes! - # TODO: Having fixed this, we also need to conslidate this if with the similar if above. - # if len(vuln_matching_fixed_packages) > 0: - if len(later_non_vuln_sibs) > 0: - - # purl_dict["TEST_test_closest_non_vulnerable"] = PackageURL.from_string( - # closest_non_vulnerable_sib.purl - # ) - # purl_dict["TEST_test_latest_non_vulnerable"] = PackageURL.from_string( - # latest_non_vulnerable_sib.purl - # ) - - purl_dict["closest_non_vulnerable"] = PackageURL.from_string( - closest_non_vulnerable_sib.purl - ) - purl_dict["latest_non_vulnerable"] = PackageURL.from_string( - latest_non_vulnerable_sib.purl - ) - - else: - - # purl_dict["TEST_test_closest_non_vulnerable"] = None - # purl_dict["TEST_test_latest_non_vulnerable"] = None - - purl_dict["closest_non_vulnerable"] = None - purl_dict["latest_non_vulnerable"] = None + # 2023-09-05 Tuesday 15:26:59. Move this chunk up to just under the start of the dict. + # # 2023-08-30 Wednesday 11:29:34. Will this handle the error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'? Yes! + # # Having fixed this, we also need to consolidate this if with the similar if above. + + # if len(later_non_vuln_sibs) > 0: + # purl_dict["closest_non_vulnerable"] = PackageURL.from_string( + # closest_non_vulnerable_sib.purl + # ) + # purl_dict["latest_non_vulnerable"] = PackageURL.from_string( + # latest_non_vulnerable_sib.purl + # ) + # else: + # purl_dict["closest_non_vulnerable"] = None + # purl_dict["latest_non_vulnerable"] = None # Temporary print output during dev/testing. from pprint import pprint diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index c1e1bff6f..6620b7e25 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -50,7 +50,11 @@ Non-vulnerable version - {{ fixed_package_details.closest_non_vulnerable.version }} + {% if fixed_package_details.closest_non_vulnerable.version %} + {{ fixed_package_details.closest_non_vulnerable.version }} + {% else %} + None. + {% endif %} + + {% if fixed_package_details.latest_non_vulnerable.version %} + {{ fixed_package_details.latest_non_vulnerable.version }} + {% else %} + None. + {% endif %} + This package is not known to be affected by vulnerabilities. {% endfor %} From 005c094828245c8f71975203fa27a6f867646c70 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Thu, 7 Sep 2023 17:58:23 -0700 Subject: [PATCH 13/53] Remove major-version code, clean comments etc. #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 245 ++++-------------- .../templates/package_details.html | 21 +- 2 files changed, 53 insertions(+), 213 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index e2a196eb4..10fd575fe 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -631,36 +631,31 @@ def get_absolute_url(self): """ return reverse("package_details", args=[self.purl]) - def get_fixed_packages(self, package): - """ - Return a queryset of all packages that fix a vulnerability with - same type, namespace, name, subpath and qualifiers of the `package` - """ - return Package.objects.filter( - name=package.name, - namespace=package.namespace, - type=package.type, - qualifiers=package.qualifiers, - subpath=package.subpath, - packagerelatedvulnerability__fix=True, - ).distinct() - - def get_sibling_packages(self, package): - """ - Return a queryset of all packages with the same type, namespace, name, subpath and - qualifiers as the target package, whether or not the 'sibling packages' fix any vulnerability. - Identical to get_fixed_packages() above except for the removal of 'packagerelatedvulnerability__fix=True,'. - """ - return Package.objects.filter( - name=package.name, - namespace=package.namespace, - type=package.type, - qualifiers=package.qualifiers, - subpath=package.subpath, - ).distinct() + def get_fixed_packages(self, package, fix=True): + """ + With fix=True, return a queryset of all packages that fix a vulnerability and have the + same type, namespace, name, subpath and qualifiers as the `package`. + With fix=False, return a queryset of all packages that have the same type, namespace, + name, subpath and qualifiers as the `package`, whether or not they fix any vulnerability. + """ + filter_dict = { + "name": package.name, + "namespace": package.namespace, + "type": package.type, + "qualifiers": package.qualifiers, + "subpath": package.subpath, + } + + if fix: + filter_dict["packagerelatedvulnerability__fix"] = True + + return Package.objects.filter(**filter_dict).distinct() def sort_by_version(self, packages_to_sort): - # Incoming is a list of objects. + """ + Use the univers version_class to sort the incoming list of vulnerabilities.models.Package + objects. + """ univers_version = RANGE_CLASS_BY_SCHEMES[packages_to_sort[0].type].version_class return sorted( packages_to_sort, @@ -670,110 +665,44 @@ def sort_by_version(self, packages_to_sort): @property def fixed_package_details(self): """ - Find and report all fixed by packages (and certain related versions) for each vulnerability in - each affected package. Report packages with no vulnerabilities as well. + For a given package, report the closest subsequent fixed by version for each of that + package's vulnerabilities as well as the closest non-vulnerable version and the most recent + non-vulnerable version. """ - # Get all fixed packages that match the target package (type etc.), regardless of fixed vuln. - matching_fixed_packages = self.get_fixed_packages(package=self) - - # Get a list of the vulnerabilities that affect the current package (i.e., self). + fixed_packages = self.get_fixed_packages(package=self, fix=True) qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - - all_sibling_packages = self.get_sibling_packages(package=self) + all_sibling_packages = self.get_fixed_packages(package=self, fix=False) non_vuln_sibs = [] for sib in all_sibling_packages: if sib.is_vulnerable is False: non_vuln_sibs.append(sib) - # Get the univers version_class for the purl, i.e., the Package the user has searched for. univers_version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class - # print("\nunivers_version_class = {}".format(univers_version_class)) - - # ======================================================= - # Get the univers version for the purl. - purl_univers_version = univers_version_class(PackageURL.from_string(self.purl).version) - # print("\n>>> purl_univers_version = {}\n".format(purl_univers_version)) - # 2023-09-04 Monday 18:23:51. Is the version just a string I can split? - # print("\n>>> type(purl_univers_version) = {}\n".format(type(purl_univers_version))) - # print( - # "\n>>> type(purl_univers_version.__str__()) = {}\n".format( - # type(purl_univers_version.__str__()) - # ) - # ) - # print( - # "\n>>> purl_univers_version.__str__().split('.')[0] = {}\n".format( - # purl_univers_version.__str__().split(".")[0] - # ) - # ) - - # Convert the univers version for the purl to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. - purl_univers_version_major_int = int(purl_univers_version.__str__().split(".")[0]) - # print("\n>>> purl_univers_version_major_int = {}\n".format(purl_univers_version_major_int)) - # print( - # "\n>>> type(purl_univers_version_major_int) = {}\n".format( - # type(purl_univers_version_major_int) - # ) - # ) - - # Create a list for the later non-vuln siblings. later_non_vuln_sibs = [] - # Each non_vuln_sib is a Package. - for non_vuln_sib in non_vuln_sibs: - - # print("\n### type(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) - # Get the univers version for the non_vuln_sib. + for non_vuln_sib in non_vuln_sibs: non_vuln_sib_univers_version = univers_version_class( PackageURL.from_string(non_vuln_sib.purl).version ) - # print("\n### non_vuln_sib_univers_version = {}\n".format(non_vuln_sib_univers_version)) - - # Convert the univers version for the purl to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. - non_vuln_sib_univers_version_major_int = int( - non_vuln_sib_univers_version.__str__().split(".")[0] - ) - - # print( - # "\n### non_vuln_sib_univers_version_major_int = {}\n".format( - # non_vuln_sib_univers_version_major_int - # ) - # ) - - # Add it to the 'later_non_vuln_sibs' list if the major versions are the same. - # if univers_version_class(non_vuln_sib.version) > univers_version_class(self.version): - if ( - univers_version_class(non_vuln_sib.version) > univers_version_class(self.version) - and purl_univers_version_major_int == non_vuln_sib_univers_version_major_int - ): + if univers_version_class(non_vuln_sib.version) > univers_version_class(self.version): later_non_vuln_sibs.append(non_vuln_sib) - # Take the list of vulnerabilities affecting the current package, retrieve a list of the fixed - # packages for each (self.get_fixed_packages()), and assign the result to a custom attribute, 'matching_fixed_packages'. - # Ex: qs[0].matching_fixed_packages returns the fixed package(s) for the 1st vulnerability. - qs = qs.prefetch_related( Prefetch( "packages", - queryset=matching_fixed_packages, - to_attr="matching_fixed_packages", + queryset=fixed_packages, + to_attr="fixed_packages", ) ) purl_dict = {} - # Pass a PackageURL object to the Jinja2 template via the context dictionary. purl_dict["purl"] = PackageURL.from_string(self.purl) - # print("\npurl = {}".format(purl_dict["purl"])) closest_non_vulnerable_sib = "" latest_non_vulnerable_sib = "" - # closest_non_vulnerable_sib_non_breaking = "" - # latest_non_vulnerable_sib_non_breaking = "" - - # We've already filled later_non_vuln_sibs with all non_vuln sibs with the same major - # version as the purl the user is searching for (i.e., self) -- but this could be an empty list. if len(later_non_vuln_sibs) > 0: closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] @@ -785,115 +714,60 @@ def fixed_package_details(self): latest_non_vulnerable_sib.purl ) - # print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) - - # later_non_vuln_sibs_non_breaking = [] - # for non_vuln_sib in later_non_vuln_sibs: - # print("\nnon_vuln_sib = {}\n".format(non_vuln_sib)) - # print("\ntype(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) - else: - # closest_non_vulnerable_sib = "" - # latest_non_vulnerable_sib = "" purl_dict["closest_non_vulnerable"] = None purl_dict["latest_non_vulnerable"] = None - # ======================================================== - # if len(later_non_vuln_sibs) > 0: - # purl_dict["closest_non_vulnerable"] = PackageURL.from_string( - # closest_non_vulnerable_sib.purl - # ) - # purl_dict["latest_non_vulnerable"] = PackageURL.from_string( - # latest_non_vulnerable_sib.purl - # ) - # else: - # purl_dict["closest_non_vulnerable"] = None - # purl_dict["latest_non_vulnerable"] = None - # ======================================================== - - # No longer need this. - # purl_univers_version_test = univers_version_class(purl_dict["purl"].version) - # print("\npurl_univers_version_test = {}".format(purl_univers_version_test)) purl_dict.update({"vulnerabilities": []}) - # For each vulnerability affecting the current Package... for vuln in qs: - later_matching_fixed_packages = [] - # Pass a Vulnerability object. purl_dict["vulnerabilities"].append({"vulnerability": vuln}) - # Using the qs prefetch, get a of all the matching fixed packages for this vulnerability, all major versions to start with. - vuln_matching_fixed_packages = vuln.matching_fixed_packages - # print( - # "\n??? type(vuln_matching_fixed_packages) = {}\n".format( - # type(vuln_matching_fixed_packages) - # ) - # ) closest_fixed_package = "" - - # ALERT: 2023-09-05 Tuesday 17:42:58. When refreshing a Package details page, I got an error: local variable 'sort_fixed_by_packages_by_version' referenced before assignment (referring to the definition of closest_fixed_package below) - # ALERT: Do I need to define it here? + closest_fixed_package_vulns = "" + later_fixed_packages = [] sort_fixed_by_packages_by_version = [] + vuln_fixed_packages = vuln.fixed_packages - if len(vuln_matching_fixed_packages) > 0: - for fixed_pkg in vuln_matching_fixed_packages: - # fixed_pkg is a 'vulnerabilities.models.Package' - # print("\n!!! type(fixed_pkg) = {}\n".format(type(fixed_pkg))) + if len(vuln_fixed_packages) > 0: + for fixed_pkg in vuln_fixed_packages: - # Get the univers version for the fixed_pkg. fixed_pkg_univers_version = univers_version_class( PackageURL.from_string(fixed_pkg.purl).version ) - # Convert the univers version for the fixed_pkg to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. - fixed_pkg_univers_version_major_int = int( - fixed_pkg_univers_version.__str__().split(".")[0] - ) + if fixed_pkg in fixed_packages and univers_version_class( + fixed_pkg.version + ) > univers_version_class(self.version): - # Add it to the 'later_non_vuln_sibs' list if the major versions are the same. - if ( - fixed_pkg in matching_fixed_packages - and univers_version_class(fixed_pkg.version) - > univers_version_class(self.version) - and purl_univers_version_major_int == fixed_pkg_univers_version_major_int - ): - later_matching_fixed_packages.append(fixed_pkg) - - # We already identified a closest fixed package and are looking for a later fixed package. - if later_matching_fixed_packages: - sort_fixed_by_packages_by_version = self.sort_by_version( - later_matching_fixed_packages - ) - # print( - # "\nsort_fixed_by_packages_by_version = {}\n".format( - # sort_fixed_by_packages_by_version - # ) - # ) + later_fixed_packages.append(fixed_pkg) - # ALERT: 2023-09-05 Tuesday 17:53:54. I think these 2 lines need to be indented to be inside the if condition above. + if later_fixed_packages: + sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) closest_fixed_package = sort_fixed_by_packages_by_version[0] closest_fixed_package_vulns = closest_fixed_package.affected_by - - # closest_fixed_package = sort_fixed_by_packages_by_version[0] - # closest_fixed_package_vulns = closest_fixed_package.affected_by + print( + "\nclosest_fixed_package_vulns = {}\n".format(closest_fixed_package_vulns) + ) + print( + "\ntype(closest_fixed_package_vulns) = {}\n".format( + type(closest_fixed_package_vulns) + ) + ) else: - # ALERT: 2023-09-05 Tuesday 18:03:46. I don't think we want this string but rather something like None, for the if condition below. - # closest_fixed_package = "There are no reported fixed packages." closest_fixed_package = None + closest_fixed_package_vulns = None for dict_vuln in purl_dict["vulnerabilities"]: if dict_vuln["vulnerability"] == vuln: - # ALERT: 2023-09-05 Tuesday 17:58:24. This is not the correct test because vuln_matching_fixed_packages might contain no major version identical to the purl major version! Instead we want to check whether the closest_fixed_package exists, which we defined above as closest_fixed_package = sort_fixed_by_packages_by_version[0] - # if len(vuln_matching_fixed_packages) > 0: if closest_fixed_package: dict_vuln["fixed_by_purl"] = PackageURL.from_string( closest_fixed_package.purl ) - # ALERT: 2023-09-05 Tuesday 18:00:09. I expect we'll also see an error here and will need to define closest_fixed_package_vulns higher above than where it is currently defined. dict_vuln["fixed_by_purl_vulnerabilities"] = [ fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] @@ -901,21 +775,6 @@ def fixed_package_details(self): dict_vuln["fixed_by_purl"] = None dict_vuln["fixed_by_purl_vulnerabilities"] = [] - # 2023-09-05 Tuesday 15:26:59. Move this chunk up to just under the start of the dict. - # # 2023-08-30 Wednesday 11:29:34. Will this handle the error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'? Yes! - # # Having fixed this, we also need to consolidate this if with the similar if above. - - # if len(later_non_vuln_sibs) > 0: - # purl_dict["closest_non_vulnerable"] = PackageURL.from_string( - # closest_non_vulnerable_sib.purl - # ) - # purl_dict["latest_non_vulnerable"] = PackageURL.from_string( - # latest_non_vulnerable_sib.purl - # ) - # else: - # purl_dict["closest_non_vulnerable"] = None - # purl_dict["latest_non_vulnerable"] = None - # Temporary print output during dev/testing. from pprint import pprint diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 6620b7e25..a14c22ade 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -47,7 +47,7 @@ - Non-vulnerable version + Next non-vulnerable version {% if fixed_package_details.closest_non_vulnerable.version %} @@ -57,21 +57,11 @@ {% endif %} - Latest non-vulnerable version - - {% if fixed_package_details.latest_non_vulnerable.version %} {{ fixed_package_details.latest_non_vulnerable.version }} {% else %} @@ -79,14 +69,6 @@ {% endif %} -
@@ -184,7 +166,6 @@ {% empty %} - This package is not known to be affected by vulnerabilities. From 55f9678ca0a95640cb42bad992290962ca25cbd6 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 11 Sep 2023 09:28:03 -0700 Subject: [PATCH 14/53] Begin test refactoring #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/tests/test_models.py | 537 ++++++++++++++++++++------- 1 file changed, 398 insertions(+), 139 deletions(-) diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index d6b17dfb1..d7615db48 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -22,6 +22,7 @@ from univers.version_range import RANGE_CLASS_BY_SCHEMES from vulnerabilities import models +from vulnerabilities.models import Alias from vulnerabilities.models import Package @@ -107,6 +108,70 @@ def test_vulnerability_package(self): @pytest.mark.django_db class TestPackageModel(TestCase): def setUp(self): + # ZAP: 2023-09-08 Friday 17:40:36. Let's start over with a focused package/vuln/fix group we know from the DB/UI testing: pkg:pypi/redis@4.1.1 + # It has 2 non-vulns versions, both the same: 5.0.0b1 + # The first of its two vulns is VCID-g2fu-45jw-aaan (aliases: CVE-2023-28858 and GHSA-24wv-mv5m-xv4h), fixed by 4.3.6 w/1 vuln of its own + # The second is VCID-rqe1-dkmg-aaad (aliases: CVE-2023-28859 and GHSA-8fww-64cx-x8p5), fixed by 5.0.0b1 w/ 0 vulns of its own + + # pkg + self.package_pypi_redis_4_1_1 = models.Package.objects.create( + type="pypi", + namespace="", + name="redix", + version="4.1.1", + qualifiers={}, + subpath="", + ) + + # vuln + self.vuln_VCID_g2fu_45jw_aaan = models.Vulnerability.objects.create( + summary="This is VCID-g2fu-45jw-aaan", + vulnerability_id="VCID-g2fu-45jw-aaan", + ) + + # relationship + models.PackageRelatedVulnerability.objects.create( + package=self.package_pypi_redis_4_1_1, + vulnerability=self.vuln_VCID_g2fu_45jw_aaan, + fix=False, + ) + + # aliases + Alias.objects.create(alias="CVE-2023-28858", vulnerability=self.vuln_VCID_g2fu_45jw_aaan) + Alias.objects.create( + alias="GHSA-24wv-mv5m-xv4h", vulnerability=self.vuln_VCID_g2fu_45jw_aaan + ) + + # ZAP: Need fix for above vuln + + # vuln + self.vuln_VCID_rqe1_dkmg_aaad = models.Vulnerability.objects.create( + summary="This is VCID-rqe1-dkmg-aaad", + vulnerability_id="VCID-rqe1-dkmg-aaad", + ) + + # relationship + models.PackageRelatedVulnerability.objects.create( + package=self.package_pypi_redis_4_1_1, + vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, + fix=False, + ) + + # aliases + Alias.objects.create(alias="CVE-2023-28859", vulnerability=self.vuln_VCID_rqe1_dkmg_aaad) + Alias.objects.create( + alias="GHSA-8fww-64cx-x8p5", vulnerability=self.vuln_VCID_rqe1_dkmg_aaad + ) + + # ZAP: Need fix for above vuln + + # ZAP: Need non-vuln version(s) + + # ======================================================== + + # ======================================================== + + # ZAP: Not sure we want to keep any of this. self.vuln1 = models.Vulnerability.objects.create( summary="test-vuln1", vulnerability_id="VCID-123", @@ -272,10 +337,58 @@ def setUp(self): ) ) + # 2023-09-08 Friday 21:24:54. New experiment + def test_explore_packages(self): + print("\nself.package_pypi_redis_4_1_1 = {}\n".format(self.package_pypi_redis_4_1_1)) + + print( + "\nself.package_pypi_redis_4_1_1.package_url = {}\n".format( + self.package_pypi_redis_4_1_1.package_url + ) + ) + + print( + "\nself.package_pypi_redis_4_1_1.plain_package_url = {}\n".format( + self.package_pypi_redis_4_1_1.plain_package_url + ) + ) + + print( + "\nself.package_pypi_redis_4_1_1.purl = {}\n".format(self.package_pypi_redis_4_1_1.purl) + ) + + print( + "\nself.package_pypi_redis_4_1_1.affected_by = {}\n".format( + self.package_pypi_redis_4_1_1.affected_by + ) + ) + + for vuln in self.package_pypi_redis_4_1_1.affected_by: + print(vuln) + print(vuln.summary) + print("") + + print( + "\nself.package_pypi_redis_4_1_1.fixing = {}\n".format( + self.package_pypi_redis_4_1_1.fixing + ) + ) + + print( + "\nself.package_pypi_redis_4_1_1.get_absolute_url() = {}\n".format( + self.package_pypi_redis_4_1_1.get_absolute_url() + ) + ) + + # ZAP: 2023-09-08 Friday 18:37:13. Need to revise this after adding new pkgs, vulns, aliases above. def test_get_vulnerable_packages(self): vuln_packages = Package.objects.vulnerable() - assert vuln_packages.count() == 3 - assert vuln_packages.distinct().count() == 2 + print("\nvuln_packages = {}\n".format(vuln_packages)) + # assert vuln_packages.count() == 3 + # 2023-09-08 Friday 16:59:33. Update given today's additions etc. + assert vuln_packages.count() == 5 + # assert vuln_packages.distinct().count() == 2 + assert vuln_packages.distinct().count() == 3 first_vulnerable_package = vuln_packages.distinct()[0] matching_fixed_packages = first_vulnerable_package.get_fixed_packages( @@ -293,33 +406,177 @@ def test_get_vulnerable_packages(self): == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1" ) - purl_dict = { - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", - "vulnerabilities": [ - { - "vulnerability": "VCID-123", - "closest_fixed_by": { - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - "type": "maven", - "namespace": "com.fasterxml.jackson.core", - "name": "jackson-databind", - "version": "2.13.2", - "qualifiers": {}, - "subpath": "", - }, - "closest_fixed_by_vulnerabilities": [{"vuln_id": "VCID-000"}], - }, - { - "vulnerability": "VCID-456", - "closest_fixed_by": {}, - "closest_fixed_by_vulnerabilities": [], - }, - ], - "closest_non_vulnerable": {}, - "latest_non_vulnerable": {}, - } - - assert vuln_packages.distinct()[0].fixed_package_details == purl_dict + # purl_dict = { + # "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + # "vulnerabilities": [ + # { + # "vulnerability": "VCID-123", + # "closest_fixed_by": { + # "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + # "type": "maven", + # "namespace": "com.fasterxml.jackson.core", + # "name": "jackson-databind", + # "version": "2.13.2", + # "qualifiers": {}, + # "subpath": "", + # }, + # "closest_fixed_by_vulnerabilities": [{"vuln_id": "VCID-000"}], + # }, + # { + # "vulnerability": "VCID-456", + # "closest_fixed_by": {}, + # "closest_fixed_by_vulnerabilities": [], + # }, + # ], + # "closest_non_vulnerable": {}, + # "latest_non_vulnerable": {}, + # } + + # purl_dict = { + # "purl": PackageURL( + # type="maven", + # namespace="com.fasterxml.jackson.core", + # name="jackson-databind", + # version="2.13.1", + # qualifiers={}, + # subpath=None, + # ), + # "closest_non_vulnerable": PackageURL( + # type="maven", + # namespace="com.fasterxml.jackson.core", + # name="jackson-databind", + # version="2.14.0-rc1", + # qualifiers={}, + # subpath=None, + # ), + # "latest_non_vulnerable": PackageURL( + # type="maven", + # namespace="com.fasterxml.jackson.core", + # name="jackson-databind", + # version="2.14.0-rc1", + # qualifiers={}, + # subpath=None, + # ), + # "vulnerabilities": [ + # { + # "vulnerability": "", + # "fixed_by_purl": PackageURL( + # type="maven", + # namespace="com.fasterxml.jackson.core", + # name="jackson-databind", + # version="2.13.2", + # qualifiers={}, + # subpath=None, + # ), + # "fixed_by_purl_vulnerabilities": [""], + # }, + # { + # "vulnerability": "", + # "fixed_by_purl": None, + # "fixed_by_purl_vulnerabilities": [], + # }, + # ], + # } + + print("\nfirst_vulnerable_package.purl = {}\n".format(first_vulnerable_package.purl)) + + print("\nfirst_vulnerable_package = {}\n".format(first_vulnerable_package)) + + assert ( + first_vulnerable_package.purl + == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" + ) + + # assert ( + # first_vulnerable_package + # # == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" + # == "" + # ) + + purl_string = "pkg:pypi/redis@4.1.1" + purl = PackageURL.from_string(purl_string) + purl_to_dict = purl.to_dict() + + # For namespace, version, qualifiers and subpath, we need to add the or * to avoid an IntegrityError, e.g., django.db.utils.IntegrityError: null value in column "subpath" violates not-null constraint + vulnerablecode_package = models.Package.objects.create( + type=purl_to_dict.get("type"), + namespace=purl_to_dict.get("namespace") or "", + name=purl_to_dict.get("name"), + version=purl_to_dict.get("version") or "", + qualifiers=purl_to_dict.get("qualifiers") or {}, + subpath=purl_to_dict.get("subpath") or "", + ) + + print("\nvulnerablecode_package = {}\n".format(vulnerablecode_package)) + print( + "\nvulnerablecode_package.fixed_package_details = {}\n".format( + vulnerablecode_package.fixed_package_details + ) + ) + + # =============================== + + # # Dictionary with class values + # my_dict = {"obj1": MyClass(1), "obj2": MyClass(2)} + + # # Print the dictionary + # print(my_dict) + + # assert vuln_packages.distinct()[0].fixed_package_details == purl_dict + + # banana = {'purl': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.13.1', qualifiers={}, subpath=None), 'closest_non_vulnerable': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.14.0-rc1', qualifiers={}, subpath=None), 'latest_non_vulnerable': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.14.0-rc1', qualifiers={}, subpath=None), 'vulnerabilities': [{'vulnerability': , 'fixed_by_purl': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.13.2', qualifiers={}, subpath=None), 'fixed_by_purl_vulnerabilities': []}, {'vulnerability': , 'fixed_by_purl': None, 'fixed_by_purl_vulnerabilities': []}]} + + # print('\nbanana = {}\n'.format(banana)) + + # assert vuln_packages.distinct()[0].`fixed_package_details` == banana + + print( + "\nvuln_packages.distinct()[0].fixed_package_details = {}\n".format( + vuln_packages.distinct()[0].fixed_package_details + ) + ) + + # print(vuln_packages.distinct()[0]["vulnerabilities"].fixed_package_details) # Error: TypeError: 'Package' object is not subscriptable + + print( + "\ntype(vuln_packages.distinct()[0].fixed_package_details) = {}\n".format( + type(vuln_packages.distinct()[0].fixed_package_details) + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details.get("purl") = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details.get("purl") + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details["purl"] = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details["purl"] + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"] = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"] + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0] = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0] + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0]["vulnerability"] = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0][ + "vulnerability" + ] + ) + ) + + print("") def test_univers_version_comparisons(self): assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") @@ -399,113 +656,115 @@ def test_sort_by_version(self): assert sorted_pkgs[0].purl == "pkg:npm/sequelize@3.9.1" assert sorted_pkgs[-1].purl == "pkg:npm/sequelize@3.40.1" - def test_string_to_purl_to_dict_to_package(self): - # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., - # a . - - # Convert a PURL string to a PURL. - purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - purl = PackageURL.from_string(purl_string) - - assert type(purl) == PackageURL - assert purl.type == "maven" - assert purl.qualifiers == {} - assert purl.subpath == None - - # Convert the PURL to a dictionary. - # ALERT: 2023-08-15 Tuesday 13:18:09. What about using the function 'def purl_to_dict(purl: PackageURL)'? Confusingly similar name but it seems designed to address the issue raised here (and looks useful for passing the data to the Jinja2 template). - # It appears that this step is where the unwanted None values are created for qualifiers and - # subpath when the PURL does not already contain values for those attributes. - purl_to_dict = purl.to_dict() - - assert purl_to_dict == { - "type": "maven", - "namespace": "org.apache.tomcat.embed", - "name": "tomcat-embed-core", - "version": "9.0.31", - "qualifiers": None, - "subpath": None, - } - assert purl_to_dict.get("qualifiers") == None - assert purl_to_dict.get("subpath") == None - - # Convert the dictionary to a VulnerableCode Package, i.e., - # a - - # If subpath is None we get error: django.db.utils.IntegrityError: null value in column - # "subpath" violates not-null constraint -- need to convert value from None to empty string. - # Similar issue with qualifiers, which must be converted from None to {}. - - # I've structured the following in this way because trying instead to use - # "with pytest.raises(IntegrityError):" will throw the error - # django.db.transaction.TransactionManagementError: An error occurred in the current - # transaction. You can't execute queries until the end of the 'atomic' block. - - try: - with transaction.atomic(): - vulnerablecode_package = models.Package.objects.create( - type=purl_to_dict.get("type"), - namespace=purl_to_dict.get("namespace"), - name=purl_to_dict.get("name"), - version=purl_to_dict.get("version"), - qualifiers=purl_to_dict.get("qualifiers"), - subpath=purl_to_dict.get("subpath"), - ) - except IntegrityError: - print("\nAs expected, an IntegrityError has occurred.\n") - - # This will avoid the IntegrityError: - if purl_to_dict.get("qualifiers") is None: - purl_to_dict["qualifiers"] = {} - if purl_to_dict.get("subpath") is None: - purl_to_dict["subpath"] = "" - - # Check the qualifiers and subpath values again. - assert purl_to_dict.get("qualifiers") == {} - assert purl_to_dict.get("subpath") == "" - - vulnerablecode_package = models.Package.objects.create( - type=purl_to_dict.get("type"), - namespace=purl_to_dict.get("namespace"), - name=purl_to_dict.get("name"), - version=purl_to_dict.get("version"), - qualifiers=purl_to_dict.get("qualifiers"), - subpath=purl_to_dict.get("subpath"), - ) - - assert type(vulnerablecode_package) == models.Package - assert ( - vulnerablecode_package.purl - == "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - ) - assert vulnerablecode_package.qualifiers == {} - assert vulnerablecode_package.subpath == "" - - def test_compare_package_major_versions(self): - # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., - # a . - - # Convert a PURL string to a PURL. - purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - purl = PackageURL.from_string(purl_string) - - assert type(purl) == PackageURL - assert purl.type == "maven" - assert purl.qualifiers == {} - assert purl.subpath == None - - print("\npurl_string = {}".format(purl_string)) - - print("\npurl = {}".format(purl)) - - print("\nHello VulnerableCode!\n") - - all_packages = Package.objects - print("\nPackage.objects = {}\n".format(Package.objects)) - print("\nall_packages.distinct() = {}\n".format(all_packages.distinct())) - print("\nall_packages.distinct()[0] = {}\n".format(all_packages.distinct()[0])) - - for pkg in all_packages.distinct(): - print(PackageURL.from_string(pkg.purl)) - - print("") + # # ZAP: 2023-09-07 Thursday 20:05:40. This has served its purpose and can be removed after a last close look. + # def test_string_to_purl_to_dict_to_package(self): + # # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., + # # a . + + # # Convert a PURL string to a PURL. + # purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + # purl = PackageURL.from_string(purl_string) + + # assert type(purl) == PackageURL + # assert purl.type == "maven" + # assert purl.qualifiers == {} + # assert purl.subpath == None + + # # Convert the PURL to a dictionary. + # # ALERT: 2023-08-15 Tuesday 13:18:09. What about using the function 'def purl_to_dict(purl: PackageURL)'? Confusingly similar name but it seems designed to address the issue raised here (and looks useful for passing the data to the Jinja2 template). + # # It appears that this step is where the unwanted None values are created for qualifiers and + # # subpath when the PURL does not already contain values for those attributes. + # purl_to_dict = purl.to_dict() + + # assert purl_to_dict == { + # "type": "maven", + # "namespace": "org.apache.tomcat.embed", + # "name": "tomcat-embed-core", + # "version": "9.0.31", + # "qualifiers": None, + # "subpath": None, + # } + # assert purl_to_dict.get("qualifiers") == None + # assert purl_to_dict.get("subpath") == None + + # # Convert the dictionary to a VulnerableCode Package, i.e., + # # a + + # # If subpath is None we get error: django.db.utils.IntegrityError: null value in column + # # "subpath" violates not-null constraint -- need to convert value from None to empty string. + # # Similar issue with qualifiers, which must be converted from None to {}. + + # # I've structured the following in this way because trying instead to use + # # "with pytest.raises(IntegrityError):" will throw the error + # # django.db.transaction.TransactionManagementError: An error occurred in the current + # # transaction. You can't execute queries until the end of the 'atomic' block. + + # try: + # with transaction.atomic(): + # vulnerablecode_package = models.Package.objects.create( + # type=purl_to_dict.get("type"), + # namespace=purl_to_dict.get("namespace"), + # name=purl_to_dict.get("name"), + # version=purl_to_dict.get("version"), + # qualifiers=purl_to_dict.get("qualifiers"), + # subpath=purl_to_dict.get("subpath"), + # ) + # except IntegrityError: + # print("\nAs expected, an IntegrityError has occurred.\n") + + # # This will avoid the IntegrityError: + # if purl_to_dict.get("qualifiers") is None: + # purl_to_dict["qualifiers"] = {} + # if purl_to_dict.get("subpath") is None: + # purl_to_dict["subpath"] = "" + + # # Check the qualifiers and subpath values again. + # assert purl_to_dict.get("qualifiers") == {} + # assert purl_to_dict.get("subpath") == "" + + # vulnerablecode_package = models.Package.objects.create( + # type=purl_to_dict.get("type"), + # namespace=purl_to_dict.get("namespace"), + # name=purl_to_dict.get("name"), + # version=purl_to_dict.get("version"), + # qualifiers=purl_to_dict.get("qualifiers"), + # subpath=purl_to_dict.get("subpath"), + # ) + + # assert type(vulnerablecode_package) == models.Package + # assert ( + # vulnerablecode_package.purl + # == "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + # ) + # assert vulnerablecode_package.qualifiers == {} + # assert vulnerablecode_package.subpath == "" + + # # ZAP: 2023-09-07 Thursday 20:32:35. Ditch this, right? + # def test_compare_package_major_versions(self): + # # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., + # # a . + + # # Convert a PURL string to a PURL. + # purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + # purl = PackageURL.from_string(purl_string) + + # assert type(purl) == PackageURL + # assert purl.type == "maven" + # assert purl.qualifiers == {} + # assert purl.subpath == None + + # print("\npurl_string = {}".format(purl_string)) + + # print("\npurl = {}".format(purl)) + + # print("\nHello VulnerableCode!\n") + + # all_packages = Package.objects + # print("\nPackage.objects = {}\n".format(Package.objects)) + # print("\nall_packages.distinct() = {}\n".format(all_packages.distinct())) + # print("\nall_packages.distinct()[0] = {}\n".format(all_packages.distinct()[0])) + + # for pkg in all_packages.distinct(): + # print(PackageURL.from_string(pkg.purl)) + + # print("") From b750d65c5ca01b40e05825286affdfc36dd0952e Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 11 Sep 2023 18:54:48 -0700 Subject: [PATCH 15/53] Finish package details code and template, refactor/create package-related tests #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 20 +- .../templates/package_details.html | 2 +- vulnerabilities/tests/test_models.py | 626 ++++-------------- 3 files changed, 149 insertions(+), 499 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 10fd575fe..2b67872a8 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -747,14 +747,6 @@ def fixed_package_details(self): sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) closest_fixed_package = sort_fixed_by_packages_by_version[0] closest_fixed_package_vulns = closest_fixed_package.affected_by - print( - "\nclosest_fixed_package_vulns = {}\n".format(closest_fixed_package_vulns) - ) - print( - "\ntype(closest_fixed_package_vulns) = {}\n".format( - type(closest_fixed_package_vulns) - ) - ) else: closest_fixed_package = None @@ -776,13 +768,13 @@ def fixed_package_details(self): dict_vuln["fixed_by_purl_vulnerabilities"] = [] # Temporary print output during dev/testing. - from pprint import pprint + # from pprint import pprint - pprint( - purl_dict, - sort_dicts=False, - ) - print("") + # pprint( + # purl_dict, + # sort_dicts=False, + # ) + # print("") return purl_dict diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 9d9c70b77..6d610feb7 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -24,7 +24,7 @@ - @@ -65,7 +65,7 @@ {% if fixed_package_details.latest_non_vulnerable.version %} {{ fixed_package_details.latest_non_vulnerable.version }} {% else %} - None. + None. {% endif %} @@ -77,7 +77,7 @@
- Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }}) + Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }})
+ ", - # "fixed_by_purl": PackageURL( - # type="maven", - # namespace="com.fasterxml.jackson.core", - # name="jackson-databind", - # version="2.13.2", - # qualifiers={}, - # subpath=None, - # ), - # "fixed_by_purl_vulnerabilities": [""], - # }, - # { - # "vulnerability": "", - # "fixed_by_purl": None, - # "fixed_by_purl_vulnerabilities": [], - # }, - # ], - # } - - print("\nfirst_vulnerable_package.purl = {}\n".format(first_vulnerable_package.purl)) - - print("\nfirst_vulnerable_package = {}\n".format(first_vulnerable_package)) - - assert ( - first_vulnerable_package.purl - == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" - ) + assert first_fixed_by_package.purl == "pkg:pypi/redis@4.3.6" - # assert ( - # first_vulnerable_package - # # == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" - # == "" - # ) + def test_string_to_package(self): - purl_string = "pkg:pypi/redis@4.1.1" + purl_string = "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" purl = PackageURL.from_string(purl_string) purl_to_dict = purl.to_dict() @@ -507,77 +337,18 @@ def test_get_vulnerable_packages(self): subpath=purl_to_dict.get("subpath") or "", ) - print("\nvulnerablecode_package = {}\n".format(vulnerablecode_package)) - print( - "\nvulnerablecode_package.fixed_package_details = {}\n".format( - vulnerablecode_package.fixed_package_details - ) - ) - - # =============================== - - # # Dictionary with class values - # my_dict = {"obj1": MyClass(1), "obj2": MyClass(2)} - - # # Print the dictionary - # print(my_dict) - - # assert vuln_packages.distinct()[0].fixed_package_details == purl_dict - - # banana = {'purl': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.13.1', qualifiers={}, subpath=None), 'closest_non_vulnerable': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.14.0-rc1', qualifiers={}, subpath=None), 'latest_non_vulnerable': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.14.0-rc1', qualifiers={}, subpath=None), 'vulnerabilities': [{'vulnerability': , 'fixed_by_purl': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.13.2', qualifiers={}, subpath=None), 'fixed_by_purl_vulnerabilities': []}, {'vulnerability': , 'fixed_by_purl': None, 'fixed_by_purl_vulnerabilities': []}]} - - # print('\nbanana = {}\n'.format(banana)) - - # assert vuln_packages.distinct()[0].`fixed_package_details` == banana - - print( - "\nvuln_packages.distinct()[0].fixed_package_details = {}\n".format( - vuln_packages.distinct()[0].fixed_package_details - ) - ) - - # print(vuln_packages.distinct()[0]["vulnerabilities"].fixed_package_details) # Error: TypeError: 'Package' object is not subscriptable - - print( - "\ntype(vuln_packages.distinct()[0].fixed_package_details) = {}\n".format( - type(vuln_packages.distinct()[0].fixed_package_details) - ) - ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details.get("purl") = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details.get("purl") - ) - ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details["purl"] = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details["purl"] - ) - ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"] = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"] - ) - ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0] = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0] - ) + assert type(vulnerablecode_package) == models.Package + assert vulnerablecode_package.purl == "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" + assert vulnerablecode_package.package_url == "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" + assert ( + vulnerablecode_package.plain_package_url + == "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0]["vulnerability"] = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0][ - "vulnerability" - ] - ) + assert ( + vulnerablecode_package.get_absolute_url() + == "/packages/pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" ) - print("") - def test_univers_version_comparisons(self): assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") @@ -655,116 +426,3 @@ def test_sort_by_version(self): assert sorted_pkgs[0].purl == "pkg:npm/sequelize@3.9.1" assert sorted_pkgs[-1].purl == "pkg:npm/sequelize@3.40.1" - - # # ZAP: 2023-09-07 Thursday 20:05:40. This has served its purpose and can be removed after a last close look. - # def test_string_to_purl_to_dict_to_package(self): - # # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., - # # a . - - # # Convert a PURL string to a PURL. - # purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - # purl = PackageURL.from_string(purl_string) - - # assert type(purl) == PackageURL - # assert purl.type == "maven" - # assert purl.qualifiers == {} - # assert purl.subpath == None - - # # Convert the PURL to a dictionary. - # # ALERT: 2023-08-15 Tuesday 13:18:09. What about using the function 'def purl_to_dict(purl: PackageURL)'? Confusingly similar name but it seems designed to address the issue raised here (and looks useful for passing the data to the Jinja2 template). - # # It appears that this step is where the unwanted None values are created for qualifiers and - # # subpath when the PURL does not already contain values for those attributes. - # purl_to_dict = purl.to_dict() - - # assert purl_to_dict == { - # "type": "maven", - # "namespace": "org.apache.tomcat.embed", - # "name": "tomcat-embed-core", - # "version": "9.0.31", - # "qualifiers": None, - # "subpath": None, - # } - # assert purl_to_dict.get("qualifiers") == None - # assert purl_to_dict.get("subpath") == None - - # # Convert the dictionary to a VulnerableCode Package, i.e., - # # a - - # # If subpath is None we get error: django.db.utils.IntegrityError: null value in column - # # "subpath" violates not-null constraint -- need to convert value from None to empty string. - # # Similar issue with qualifiers, which must be converted from None to {}. - - # # I've structured the following in this way because trying instead to use - # # "with pytest.raises(IntegrityError):" will throw the error - # # django.db.transaction.TransactionManagementError: An error occurred in the current - # # transaction. You can't execute queries until the end of the 'atomic' block. - - # try: - # with transaction.atomic(): - # vulnerablecode_package = models.Package.objects.create( - # type=purl_to_dict.get("type"), - # namespace=purl_to_dict.get("namespace"), - # name=purl_to_dict.get("name"), - # version=purl_to_dict.get("version"), - # qualifiers=purl_to_dict.get("qualifiers"), - # subpath=purl_to_dict.get("subpath"), - # ) - # except IntegrityError: - # print("\nAs expected, an IntegrityError has occurred.\n") - - # # This will avoid the IntegrityError: - # if purl_to_dict.get("qualifiers") is None: - # purl_to_dict["qualifiers"] = {} - # if purl_to_dict.get("subpath") is None: - # purl_to_dict["subpath"] = "" - - # # Check the qualifiers and subpath values again. - # assert purl_to_dict.get("qualifiers") == {} - # assert purl_to_dict.get("subpath") == "" - - # vulnerablecode_package = models.Package.objects.create( - # type=purl_to_dict.get("type"), - # namespace=purl_to_dict.get("namespace"), - # name=purl_to_dict.get("name"), - # version=purl_to_dict.get("version"), - # qualifiers=purl_to_dict.get("qualifiers"), - # subpath=purl_to_dict.get("subpath"), - # ) - - # assert type(vulnerablecode_package) == models.Package - # assert ( - # vulnerablecode_package.purl - # == "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - # ) - # assert vulnerablecode_package.qualifiers == {} - # assert vulnerablecode_package.subpath == "" - - # # ZAP: 2023-09-07 Thursday 20:32:35. Ditch this, right? - # def test_compare_package_major_versions(self): - # # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., - # # a . - - # # Convert a PURL string to a PURL. - # purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - # purl = PackageURL.from_string(purl_string) - - # assert type(purl) == PackageURL - # assert purl.type == "maven" - # assert purl.qualifiers == {} - # assert purl.subpath == None - - # print("\npurl_string = {}".format(purl_string)) - - # print("\npurl = {}".format(purl)) - - # print("\nHello VulnerableCode!\n") - - # all_packages = Package.objects - # print("\nPackage.objects = {}\n".format(Package.objects)) - # print("\nall_packages.distinct() = {}\n".format(all_packages.distinct())) - # print("\nall_packages.distinct()[0] = {}\n".format(all_packages.distinct()[0])) - - # for pkg in all_packages.distinct(): - # print(PackageURL.from_string(pkg.purl)) - - # print("") From fe23fecf235683b86a5072d8735e93745e3daccd Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 18 Sep 2023 12:17:39 -0700 Subject: [PATCH 16/53] Commit the initial refactoring changes from last week #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/api.py | 160 ++++++++++++++++++++++++++- vulnerabilities/models.py | 120 ++++++++++---------- vulnerabilities/tests/test_models.py | 5 +- 3 files changed, 221 insertions(+), 64 deletions(-) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 2c8e913ba..ce862f00a 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -46,11 +46,87 @@ class MinimalPackageSerializer(serializers.HyperlinkedModelSerializer): Used for nesting inside vulnerability focused APIs. """ + affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities") + + def get_affected_vulnerabilities(self, package): + # HOT: 2023-09-13 Wednesday 13:48:01. 'parent_affected_vulnerabilities' identifies (1) the vulnerabilities (if any) that affect the given package that itself fixes one of the target purl's vulnerabilities, and (2) + # Ex: + # package.purl = pkg:pypi/redis@4.3.6 + # parent_affected_vulnerabilities = [{'vulnerability': , 'fixed_by_purl': PackageURL(type='pypi', namespace=None, name='redis', version='5.0.0b1', qualifiers={}, subpath=None), 'fixed_by_purl_vulnerabilities': []}] + + parent_affected_vulnerabilities = package.fixed_package_details.get("vulnerabilities", None) + affected_vulnerabilities = [] + + print("\npackage.purl = {}".format(package.purl)) + print("\nparent_affected_vulnerabilities = {}\n".format(parent_affected_vulnerabilities)) + # print( + # "\nparent_affected_vulnerabilities.get('vulnerability', None) = {}\n".format( + # parent_affected_vulnerabilities.get("vulnerability", None) + # ) + # ) + # print( + # "\nparent_affected_vulnerabilities.get('fixed_by_purl', None) = {}\n".format( + # parent_affected_vulnerabilities.get("fixed_by_purl", None) + # ) + # ) + # print( + # "\nparent_affected_vulnerabilities.get('fixed_by_purl_vulnerabilities', None) = {}\n".format( + # parent_affected_vulnerabilities.get("fixed_by_purl_vulnerabilities", None) + # ) + # ) + + if parent_affected_vulnerabilities: + for vuln in parent_affected_vulnerabilities: + affected_vulnerability = {} + + print( + '\nvuln.get("vulnerability", None) = {}'.format(vuln.get("vulnerability", None)) + ) + affected_vulnerability["vulnerability"] = vuln.get( + "vulnerability", None + ).vulnerability_id + + print( + '\nvuln.get("fixed_by_purl", None) = {}'.format(vuln.get("fixed_by_purl", None)) + ) + affected_vulnerability["fixed_by_purl"] = vuln.get( + "fixed_by_purl", None + ).to_string() + + print( + '\nvuln.get("fixed_by_purl_vulnerabilities", None) = {}'.format( + vuln.get("fixed_by_purl_vulnerabilities", None) + ) + ) + affected_vulnerability["fixed_by_purl_vulnerabilities"] = [ + fixed_by_purl_vulnerability.vulnerability_id + for fixed_by_purl_vulnerability in vuln.get( + "fixed_by_purl_vulnerabilities", None + ) + ] + + affected_vulnerabilities.append(affected_vulnerability) + + print("affected_vulnerability = {}".format(affected_vulnerability)) + print("affected_vulnerabilities = {}".format(affected_vulnerabilities)) + + return affected_vulnerabilities + + # ZAP: 2023-09-12 Tuesday 19:12:22. Not what we want but this does return the purl, which we can use to find all of its own vulns (if any). + # return package.purl + + # def get_affected_vulnerabilities_saved(self, package) -> dict: + # """ + # Return a mapping of vulnerabilities that affects the given `package`. + # """ + # # return self.get_vulnerabilities_for_a_package(package=package, fix=False) + # return self.get_vulnerabilities_for_a_package(package=self, fix=False) + purl = serializers.CharField(source="package_url") class Meta: model = Package - fields = ["url", "purl", "is_vulnerable"] + fields = ["url", "purl", "is_vulnerable", "affected_by_vulnerabilities"] class MinimalVulnerabilitySerializer(serializers.HyperlinkedModelSerializer): @@ -91,9 +167,41 @@ def to_representation(self, instance): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") + # ZAP: 2023-09-14 Thursday 11:34:26. Add closest fixed package from the Package UI here. + + closest_fixed_package = serializers.SerializerMethodField("get_closest_fixed_package") + + def get_closest_fixed_package(self, instance): + return "Coming soon!" + + # test_data = super().to_representation(instance) + # vulnerabilities = [vuln["vulnerability"] for vuln in test_data["vulnerabilities"]] + # test_data["vulnerabilities"] = vulnerabilities + + # return test_data + + # # ZAP: From the PackageSerializer class: + # # next_non_vulnerable_version = serializers.SerializerMethodField("get_closest_non_vulnerable") + + # def get_closest_non_vulnerable(self, package): + # closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) + # if closest_non_vulnerable: + # return closest_non_vulnerable.version + # else: + # return None + class Meta: model = Vulnerability - fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] + # fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] + fields = [ + "url", + "vulnerability_id", + "summary", + "references", + "fixed_packages", + "closest_fixed_package", + "aliases", + ] class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer): @@ -106,8 +214,34 @@ class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") + # ALERT: 2023-09-14 Thursday 15:16:05. For some reason, favorite_color is not appearing in the output. + # favorite_color = serializers.SerializerMethodField("get_favorite_color") + + # # def get_closest_non_vulnerable(self, package): + # def get_favorite_color(self, package): + # # closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) + # # if closest_non_vulnerable: + # # return closest_non_vulnerable.version + # # else: + # # return None + # return "red!" + + favorite_color = MinimalPackageSerializer( + many=True, source="filtered_fixed_packages", read_only=True + ) + class Meta: model = Vulnerability + # fields = [ + # "url", + # "vulnerability_id", + # "summary", + # "aliases", + # "fixed_packages", + # "affected_packages", + # "references", + # ] + fields = [ "url", "vulnerability_id", @@ -116,6 +250,7 @@ class Meta: "fixed_packages", "affected_packages", "references", + "favorite_color", ] @@ -124,6 +259,24 @@ class PackageSerializer(serializers.HyperlinkedModelSerializer): Lookup software package using Package URLs """ + next_non_vulnerable_version = serializers.SerializerMethodField("get_closest_non_vulnerable") + + def get_closest_non_vulnerable(self, package): + closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) + if closest_non_vulnerable: + return closest_non_vulnerable.version + else: + return None + + latest_non_vulnerable_version = serializers.SerializerMethodField("get_latest_non_vulnerable") + + def get_latest_non_vulnerable(self, package): + latest_non_vulnerable = package.fixed_package_details.get("latest_non_vulnerable", None) + if latest_non_vulnerable: + return latest_non_vulnerable.version + else: + return None + purl = serializers.CharField(source="package_url") affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities") @@ -159,6 +312,7 @@ def get_vulnerabilities_for_a_package(self, package, fix) -> dict: to_attr="filtered_fixed_packages", ) ) + # so this calls VulnSerializerRefsAndSummary return VulnSerializerRefsAndSummary( instance=qs, many=True, @@ -190,6 +344,8 @@ class Meta: "subpath", "affected_by_vulnerabilities", "fixing_vulnerabilities", + "next_non_vulnerable_version", + "latest_non_vulnerable_version", ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 2b67872a8..76b0dbbfd 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -631,19 +631,17 @@ def get_absolute_url(self): """ return reverse("package_details", args=[self.purl]) - def get_fixed_packages(self, package, fix=True): + def get_fixing_package_versions(self, fix=True): """ - With fix=True, return a queryset of all packages that fix a vulnerability and have the - same type, namespace, name, subpath and qualifiers as the `package`. - With fix=False, return a queryset of all packages that have the same type, namespace, - name, subpath and qualifiers as the `package`, whether or not they fix any vulnerability. + Return a queryset of all the package versions of this `package` that fix any vulnerability. + If `fix` is False, return all package versions whether or not they fix a vulnerability. """ filter_dict = { - "name": package.name, - "namespace": package.namespace, - "type": package.type, - "qualifiers": package.qualifiers, - "subpath": package.subpath, + "name": self.name, + "namespace": self.namespace, + "type": self.type, + "qualifiers": self.qualifiers, + "subpath": self.subpath, } if fix: @@ -651,15 +649,14 @@ def get_fixed_packages(self, package, fix=True): return Package.objects.filter(**filter_dict).distinct() - def sort_by_version(self, packages_to_sort): + def sort_by_version(self, packages): """ - Use the univers version_class to sort the incoming list of vulnerabilities.models.Package - objects. + Return a list of `packages` sorted by version. """ - univers_version = RANGE_CLASS_BY_SCHEMES[packages_to_sort[0].type].version_class + version_class = RANGE_CLASS_BY_SCHEMES[packages[0].type].version_class return sorted( - packages_to_sort, - key=lambda x: univers_version(x.version), + packages, + key=lambda x: version_class(x.version), ) @property @@ -669,60 +666,61 @@ def fixed_package_details(self): package's vulnerabilities as well as the closest non-vulnerable version and the most recent non-vulnerable version. """ - fixed_packages = self.get_fixed_packages(package=self, fix=True) - qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - all_sibling_packages = self.get_fixed_packages(package=self, fix=False) - - non_vuln_sibs = [] - for sib in all_sibling_packages: - if sib.is_vulnerable is False: - non_vuln_sibs.append(sib) + fixing_packages = self.get_fixing_package_versions(fix=True) + package_vulnerabilities = self.vulnerabilities.filter( + packagerelatedvulnerability__fix=False + ) + package_versions = self.get_fixing_package_versions(fix=False) - univers_version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class - later_non_vuln_sibs = [] + non_vulnerable_versions = [] + for version in package_versions: + if not version.is_vulnerable: + non_vulnerable_versions.append(version) - for non_vuln_sib in non_vuln_sibs: - non_vuln_sib_univers_version = univers_version_class( - PackageURL.from_string(non_vuln_sib.purl).version - ) + version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class + later_non_vulnerable_versions = [] - if univers_version_class(non_vuln_sib.version) > univers_version_class(self.version): - later_non_vuln_sibs.append(non_vuln_sib) + current_version = version_class(self.version) + for non_vuln_ver in non_vulnerable_versions: + if version_class(non_vuln_ver.version) > current_version: + later_non_vulnerable_versions.append(non_vuln_ver) - qs = qs.prefetch_related( + package_vulnerabilities = package_vulnerabilities.prefetch_related( Prefetch( "packages", - queryset=fixed_packages, + queryset=fixing_packages, to_attr="fixed_packages", ) ) - purl_dict = {} - purl_dict["purl"] = PackageURL.from_string(self.purl) + package_details = {} + package_details["purl"] = PackageURL.from_string(self.purl) - closest_non_vulnerable_sib = "" - latest_non_vulnerable_sib = "" + closest_non_vulnerable_version = "" + latest_non_vulnerable_version = "" - if len(later_non_vuln_sibs) > 0: - closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] + # ZAP: 2023-09-18 Monday 11:40:05. I think Philippe and I got up to this point during last Thursday's code review. We also reviewed much of the API structure and output which still needs more updating -- and then some tests. - purl_dict["closest_non_vulnerable"] = PackageURL.from_string( - closest_non_vulnerable_sib.purl + if later_non_vulnerable_versions: + closest_non_vulnerable_version = self.sort_by_version(later_non_vulnerable_versions)[0] + latest_non_vulnerable_version = self.sort_by_version(later_non_vulnerable_versions)[-1] + + package_details["closest_non_vulnerable"] = PackageURL.from_string( + closest_non_vulnerable_version.purl ) - purl_dict["latest_non_vulnerable"] = PackageURL.from_string( - latest_non_vulnerable_sib.purl + package_details["latest_non_vulnerable"] = PackageURL.from_string( + latest_non_vulnerable_version.purl ) else: - purl_dict["closest_non_vulnerable"] = None - purl_dict["latest_non_vulnerable"] = None + package_details["closest_non_vulnerable"] = None + package_details["latest_non_vulnerable"] = None - purl_dict.update({"vulnerabilities": []}) + package_details.update({"vulnerabilities": []}) - for vuln in qs: - purl_dict["vulnerabilities"].append({"vulnerability": vuln}) + for vuln in package_vulnerabilities: + package_details["vulnerabilities"].append({"vulnerability": vuln}) closest_fixed_package = "" closest_fixed_package_vulns = "" @@ -733,13 +731,13 @@ def fixed_package_details(self): if len(vuln_fixed_packages) > 0: for fixed_pkg in vuln_fixed_packages: - fixed_pkg_univers_version = univers_version_class( + fixed_pkg_univers_version = version_class( PackageURL.from_string(fixed_pkg.purl).version ) - if fixed_pkg in fixed_packages and univers_version_class( + if fixed_pkg in fixing_packages and version_class( fixed_pkg.version - ) > univers_version_class(self.version): + ) > version_class(self.version): later_fixed_packages.append(fixed_pkg) @@ -752,7 +750,7 @@ def fixed_package_details(self): closest_fixed_package = None closest_fixed_package_vulns = None - for dict_vuln in purl_dict["vulnerabilities"]: + for dict_vuln in package_details["vulnerabilities"]: if dict_vuln["vulnerability"] == vuln: if closest_fixed_package: @@ -768,15 +766,15 @@ def fixed_package_details(self): dict_vuln["fixed_by_purl_vulnerabilities"] = [] # Temporary print output during dev/testing. - # from pprint import pprint + from pprint import pprint - # pprint( - # purl_dict, - # sort_dicts=False, - # ) - # print("") + pprint( + package_details, + sort_dicts=False, + ) + print("") - return purl_dict + return package_details class PackageRelatedVulnerability(models.Model): diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 3b560c26d..16979e0a3 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -312,7 +312,10 @@ def test_get_vulnerable_packages(self): assert vuln_packages.distinct().count() == 2 first_vulnerable_package = vuln_packages.distinct()[0] - matching_fixed_packages = first_vulnerable_package.get_fixed_packages( + # matching_fixed_packages = first_vulnerable_package.get_fixed_packages( + # first_vulnerable_package + # ) + matching_fixed_packages = first_vulnerable_package.get_fixing_package_versions( first_vulnerable_package ) first_fixed_by_package = matching_fixed_packages[0] From 89aab8f5c797c73ca13e236c466e93cfa454f63d Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Thu, 21 Sep 2023 12:44:44 -0700 Subject: [PATCH 17/53] Refactor package details-related code #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 178 +++++++++--------- .../templates/package_details.html | 24 +-- vulnerabilities/templates/packages.html | 6 +- .../templates/vulnerabilities.html | 4 +- .../templates/vulnerability_details.html | 16 +- vulnerabilities/tests/test_models.py | 30 ++- vulnerablecode/static/css/custom.css | 16 ++ 7 files changed, 137 insertions(+), 137 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 76b0dbbfd..17b7af78f 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -590,7 +590,6 @@ def __str__(self): return self.package_url @property - # TODO: consider renaming to "affected_by" def affected_by(self): """ Return a queryset of vulnerabilities affecting this package. @@ -631,7 +630,8 @@ def get_absolute_url(self): """ return reverse("package_details", args=[self.purl]) - def get_fixing_package_versions(self, fix=True): + # TODO: There are other methods, variables etc. in models.py that need similar renaming. + def get_fixed_by_package_versions(self, fix=True): """ Return a queryset of all the package versions of this `package` that fix any vulnerability. If `fix` is False, return all package versions whether or not they fix a vulnerability. @@ -653,128 +653,118 @@ def sort_by_version(self, packages): """ Return a list of `packages` sorted by version. """ + if not packages: + return [] + version_class = RANGE_CLASS_BY_SCHEMES[packages[0].type].version_class return sorted( packages, key=lambda x: version_class(x.version), ) + @property + def version_class(self): + return RANGE_CLASS_BY_SCHEMES[self.type].version_class + + @property + def current_version(self): + return self.version_class(self.version) + @property def fixed_package_details(self): """ - For a given package, report the closest subsequent fixed by version for each of that - package's vulnerabilities as well as the closest non-vulnerable version and the most recent - non-vulnerable version. + Return a mapping of vulnerabilities that affect this package and the closest and + latest non-vulnerable versions. """ - fixing_packages = self.get_fixing_package_versions(fix=True) - package_vulnerabilities = self.vulnerabilities.filter( - packagerelatedvulnerability__fix=False - ) - package_versions = self.get_fixing_package_versions(fix=False) + package_details = {} + package_details["purl"] = PackageURL.from_string(self.purl) + + closest_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() + package_details["closest_non_vulnerable"] = closest_non_vulnerable + package_details["latest_non_vulnerable"] = latest_non_vulnerable + + package_details["vulnerabilities"] = self.get_affecting_vulnerabilities() + + return package_details + + def get_non_vulnerable_versions(self): + """ + Return a tuple of the closest and latest non-vulnerable versions as PackageURLs. Return a tuple of + (None, None) if there is no non-vulnerable version. + """ + package_versions = self.get_fixed_by_package_versions(fix=False) non_vulnerable_versions = [] for version in package_versions: if not version.is_vulnerable: non_vulnerable_versions.append(version) - version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class later_non_vulnerable_versions = [] - - current_version = version_class(self.version) for non_vuln_ver in non_vulnerable_versions: - if version_class(non_vuln_ver.version) > current_version: + if self.version_class(non_vuln_ver.version) > self.current_version: later_non_vulnerable_versions.append(non_vuln_ver) - package_vulnerabilities = package_vulnerabilities.prefetch_related( - Prefetch( - "packages", - queryset=fixing_packages, - to_attr="fixed_packages", - ) - ) - - package_details = {} - package_details["purl"] = PackageURL.from_string(self.purl) - - closest_non_vulnerable_version = "" - latest_non_vulnerable_version = "" + if later_non_vulnerable_versions: + sorted_versions = self.sort_by_version(later_non_vulnerable_versions) + closest_non_vulnerable_version = sorted_versions[0] + latest_non_vulnerable_version = sorted_versions[-1] - # ZAP: 2023-09-18 Monday 11:40:05. I think Philippe and I got up to this point during last Thursday's code review. We also reviewed much of the API structure and output which still needs more updating -- and then some tests. + closest_non_vulnerable = PackageURL.from_string(closest_non_vulnerable_version.purl) + latest_non_vulnerable = PackageURL.from_string(latest_non_vulnerable_version.purl) - if later_non_vulnerable_versions: - closest_non_vulnerable_version = self.sort_by_version(later_non_vulnerable_versions)[0] - latest_non_vulnerable_version = self.sort_by_version(later_non_vulnerable_versions)[-1] + return closest_non_vulnerable, latest_non_vulnerable - package_details["closest_non_vulnerable"] = PackageURL.from_string( - closest_non_vulnerable_version.purl - ) - package_details["latest_non_vulnerable"] = PackageURL.from_string( - latest_non_vulnerable_version.purl - ) + return None, None - else: + def get_affecting_vulnerabilities(self): + """ + Return a list of vulnerabilities that affect this package together with information regarding + the versions that fix the vulnerabilities. + """ + package_details_vulns = [] - package_details["closest_non_vulnerable"] = None - package_details["latest_non_vulnerable"] = None + fixed_by_packages = self.get_fixed_by_package_versions(fix=True) - package_details.update({"vulnerabilities": []}) + package_vulnerabilities = self.vulnerabilities.filter( + packagerelatedvulnerability__fix=False + ).prefetch_related( + Prefetch( + "packages", + queryset=fixed_by_packages, + to_attr="fixed_packages", + ) + ) for vuln in package_vulnerabilities: - package_details["vulnerabilities"].append({"vulnerability": vuln}) - - closest_fixed_package = "" - closest_fixed_package_vulns = "" + package_details_vulns.append({"vulnerability": vuln}) later_fixed_packages = [] - sort_fixed_by_packages_by_version = [] - vuln_fixed_packages = vuln.fixed_packages - if len(vuln_fixed_packages) > 0: - for fixed_pkg in vuln_fixed_packages: - - fixed_pkg_univers_version = version_class( - PackageURL.from_string(fixed_pkg.purl).version - ) - - if fixed_pkg in fixing_packages and version_class( - fixed_pkg.version - ) > version_class(self.version): - - later_fixed_packages.append(fixed_pkg) - - if later_fixed_packages: - sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) - closest_fixed_package = sort_fixed_by_packages_by_version[0] - closest_fixed_package_vulns = closest_fixed_package.affected_by - - else: - closest_fixed_package = None - closest_fixed_package_vulns = None - - for dict_vuln in package_details["vulnerabilities"]: - if dict_vuln["vulnerability"] == vuln: - - if closest_fixed_package: - dict_vuln["fixed_by_purl"] = PackageURL.from_string( - closest_fixed_package.purl - ) - - dict_vuln["fixed_by_purl_vulnerabilities"] = [ - fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns - ] - else: - dict_vuln["fixed_by_purl"] = None - dict_vuln["fixed_by_purl_vulnerabilities"] = [] - - # Temporary print output during dev/testing. - from pprint import pprint - - pprint( - package_details, - sort_dicts=False, - ) - print("") - - return package_details + for fixed_pkg in vuln.fixed_packages: + if fixed_pkg not in fixed_by_packages: + continue + fixed_version = self.version_class(fixed_pkg.version) + if fixed_version > self.current_version: + later_fixed_packages.append(fixed_pkg) + + closest_fixed_package = None + closest_fixed_package_vulns = [] + + if later_fixed_packages: + sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) + closest_fixed_package = sort_fixed_by_packages_by_version[0] + fixed_by_purl = PackageURL.from_string(closest_fixed_package.purl) + closest_fixed_package_vulns = list(closest_fixed_package.affected_by) + + for vuln_details in package_details_vulns: + if vuln_details["vulnerability"] != vuln: + continue + vuln_details["fixed_by_purl"] = None + vuln_details["fixed_by_purl_vulnerabilities"] = [] + if closest_fixed_package: + vuln_details["fixed_by_purl"] = fixed_by_purl + vuln_details["fixed_by_purl_vulnerabilities"] = closest_fixed_package_vulns + + return package_details_vulns class PackageRelatedVulnerability(models.Model): diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 6d610feb7..246b3733e 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -53,7 +53,7 @@ {% if fixed_package_details.closest_non_vulnerable.version %} {{ fixed_package_details.closest_non_vulnerable.version }} {% else %} - None. + None. {% endif %}
@@ -85,7 +85,7 @@ - + @@ -117,20 +117,20 @@ {% for vuln in value %} {% if vuln.vulnerability.vulnerability_id == vulnerability.vulnerability_id %} {% if vuln.fixed_by_purl is None %} - There are no reported fixed versions. + There are no reported fixed versions. {% else %} {% if vuln.fixed_by_purl_vulnerabilities|length == 0 %} {{ vuln.fixed_by_purl.version }}
- Affected by 0 other vulnerabilities. + Affected by 0 other vulnerabilities. {% else %} {{ vuln.fixed_by_purl.version }} {% if vuln.fixed_by_purl_vulnerabilities|length != 1 %}
- Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. {% else %}
- Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerability. + Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerability. {% endif %}
@@ -141,7 +141,7 @@
- This version is affected by these other vulnerabilities: + This version is affected by these other vulnerabilities:
{% for fixed_by_vuln in vuln.fixed_by_purl_vulnerabilities %}
@@ -166,7 +166,7 @@ {% empty %}
{% endfor %} @@ -176,7 +176,7 @@
- Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }}) + Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }})
Vulnerability SummaryFixed byFixed by
- This package is not known to be affected by vulnerabilities. + This package is not known to be affected by vulnerabilities.
@@ -211,7 +211,7 @@ {% empty %} {% endfor %} diff --git a/vulnerabilities/templates/packages.html b/vulnerabilities/templates/packages.html index 000fe45be..357d60ce7 100644 --- a/vulnerabilities/templates/packages.html +++ b/vulnerabilities/templates/packages.html @@ -24,7 +24,7 @@ - +
- This package is not known to fix vulnerabilities. + This package is not known to fix vulnerabilities.
@@ -41,14 +41,14 @@ - Affected by vulnerabilities + Affected by vulnerabilities diff --git a/vulnerabilities/templates/vulnerabilities.html b/vulnerabilities/templates/vulnerabilities.html index 023d3f97f..bdada6ee1 100644 --- a/vulnerabilities/templates/vulnerabilities.html +++ b/vulnerabilities/templates/vulnerabilities.html @@ -32,8 +32,8 @@ - - + + diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 23eb455e8..d81bd71ab 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -34,14 +34,14 @@
  • - Fixed by packages ({{ fixed_by_packages|length }}) + Fixed by packages ({{ fixed_by_packages|length }})
  • - Affected packages ({{ affected_packages|length }}) + Affected packages ({{ affected_packages|length }})
    • @@ -124,7 +124,7 @@
    - Fixed by packages ({{ fixed_by_packages|length }}) + Fixed by packages ({{ fixed_by_packages|length }})
    - Fixing vulnerabilities + Fixed by vulnerabilities
    Vulnerability id AliasesAffected packagesFixed by packagesAffected packagesFixed by packages
    @@ -138,14 +138,14 @@ {% empty %} {% endfor %} {% if fixed_by_packages|length > 3 %} {% endif %} @@ -153,7 +153,7 @@
    - Affected packages ({{ affected_packages|length }}) + Affected packages ({{ affected_packages|length }})
    - There are no known fixed packages. + There are no known fixed by packages.
    - See Fixed by packages tab for more + See Fixed by packages tab for more
    @@ -167,14 +167,14 @@ {% empty %} {% endfor %} {% if affected_packages|length > 3 %} {% endif %} diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 16979e0a3..8f8b99b29 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -227,22 +227,16 @@ def test_fixed_package_details(self): searched_for_package_details = searched_for_package.fixed_package_details - purl_dict = { + package_details = { "purl": PackageURL( type="pypi", - namespace=None, name="redis", version="4.1.1", - qualifiers={}, - subpath=None, ), "closest_non_vulnerable": PackageURL( type="pypi", - namespace=None, name="redis", version="5.0.0b1", - qualifiers={}, - subpath=None, ), "latest_non_vulnerable": PackageURL( type="pypi", @@ -280,7 +274,7 @@ def test_fixed_package_details(self): ], } - assert searched_for_package_details == purl_dict + assert searched_for_package_details == package_details assert searched_for_package_details.get("closest_non_vulnerable") == PackageURL( type="pypi", @@ -300,11 +294,11 @@ def test_fixed_package_details(self): subpath=None, ) - qs_searched_for_package_fixing = searched_for_package.fixing - assert type(qs_searched_for_package_fixing) == models.VulnerabilityQuerySet - assert qs_searched_for_package_fixing.count() == 0 - assert len(qs_searched_for_package_fixing) == 0 - assert list(qs_searched_for_package_fixing) == [] + searched_for_package_fixing = searched_for_package.fixing + assert type(searched_for_package_fixing) == models.VulnerabilityQuerySet + assert searched_for_package_fixing.count() == 0 + assert len(searched_for_package_fixing) == 0 + assert list(searched_for_package_fixing) == [] def test_get_vulnerable_packages(self): vuln_packages = Package.objects.vulnerable() @@ -312,10 +306,8 @@ def test_get_vulnerable_packages(self): assert vuln_packages.distinct().count() == 2 first_vulnerable_package = vuln_packages.distinct()[0] - # matching_fixed_packages = first_vulnerable_package.get_fixed_packages( - # first_vulnerable_package - # ) - matching_fixed_packages = first_vulnerable_package.get_fixing_package_versions( + + matching_fixed_packages = first_vulnerable_package.get_fixed_by_package_versions( first_vulnerable_package ) first_fixed_by_package = matching_fixed_packages[0] @@ -330,7 +322,9 @@ def test_string_to_package(self): purl = PackageURL.from_string(purl_string) purl_to_dict = purl.to_dict() - # For namespace, version, qualifiers and subpath, we need to add the or * to avoid an IntegrityError, e.g., django.db.utils.IntegrityError: null value in column "subpath" violates not-null constraint + # For namespace, version, qualifiers and subpath, we need to add the or * to avoid an + # IntegrityError, e.g., django.db.utils.IntegrityError: null value in column "subpath" violates + # not-null constraint vulnerablecode_package = models.Package.objects.create( type=purl_to_dict.get("type"), namespace=purl_to_dict.get("namespace") or "", diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index 597432b12..f141fb62e 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -426,3 +426,19 @@ span.tag.custom { border: solid 1px #dbdbdb; background-color: #ffffff; } + +/* Emphasis for affected/fixed headings and related references. */ +.affected-fixed { + color: #ff0000; + font-weight: 700; +} + +/* Emphasis for not vulnerable. */ +.emphasis-not-vulnerable { + background-color: #e6ffe6; +} + +/* Emphasis for vulnerable. */ +.emphasis-vulnerable { + background-color: #ffe6e6; +} From e2e4e139d0553804877e78f51953e292dd7540e9 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 26 Sep 2023 19:02:56 -0700 Subject: [PATCH 18/53] Update Package details UI and Package API #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/api.py | 140 ++---------------- vulnerabilities/models.py | 42 ++++-- .../templates/package_details.html | 70 ++++----- vulnerablecode/settings.py | 1 - vulnerablecode/static/css/custom.css | 5 +- 5 files changed, 77 insertions(+), 181 deletions(-) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index ce862f00a..2ccc143e8 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -49,84 +49,26 @@ class MinimalPackageSerializer(serializers.HyperlinkedModelSerializer): affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities") def get_affected_vulnerabilities(self, package): - # HOT: 2023-09-13 Wednesday 13:48:01. 'parent_affected_vulnerabilities' identifies (1) the vulnerabilities (if any) that affect the given package that itself fixes one of the target purl's vulnerabilities, and (2) - # Ex: - # package.purl = pkg:pypi/redis@4.3.6 - # parent_affected_vulnerabilities = [{'vulnerability': , 'fixed_by_purl': PackageURL(type='pypi', namespace=None, name='redis', version='5.0.0b1', qualifiers={}, subpath=None), 'fixed_by_purl_vulnerabilities': []}] - parent_affected_vulnerabilities = package.fixed_package_details.get("vulnerabilities", None) affected_vulnerabilities = [] - print("\npackage.purl = {}".format(package.purl)) - print("\nparent_affected_vulnerabilities = {}\n".format(parent_affected_vulnerabilities)) - # print( - # "\nparent_affected_vulnerabilities.get('vulnerability', None) = {}\n".format( - # parent_affected_vulnerabilities.get("vulnerability", None) - # ) - # ) - # print( - # "\nparent_affected_vulnerabilities.get('fixed_by_purl', None) = {}\n".format( - # parent_affected_vulnerabilities.get("fixed_by_purl", None) - # ) - # ) - # print( - # "\nparent_affected_vulnerabilities.get('fixed_by_purl_vulnerabilities', None) = {}\n".format( - # parent_affected_vulnerabilities.get("fixed_by_purl_vulnerabilities", None) - # ) - # ) - if parent_affected_vulnerabilities: for vuln in parent_affected_vulnerabilities: affected_vulnerability = {} - print( - '\nvuln.get("vulnerability", None) = {}'.format(vuln.get("vulnerability", None)) - ) affected_vulnerability["vulnerability"] = vuln.get( "vulnerability", None ).vulnerability_id - print( - '\nvuln.get("fixed_by_purl", None) = {}'.format(vuln.get("fixed_by_purl", None)) - ) - affected_vulnerability["fixed_by_purl"] = vuln.get( - "fixed_by_purl", None - ).to_string() - - print( - '\nvuln.get("fixed_by_purl_vulnerabilities", None) = {}'.format( - vuln.get("fixed_by_purl_vulnerabilities", None) - ) - ) - affected_vulnerability["fixed_by_purl_vulnerabilities"] = [ - fixed_by_purl_vulnerability.vulnerability_id - for fixed_by_purl_vulnerability in vuln.get( - "fixed_by_purl_vulnerabilities", None - ) - ] - affected_vulnerabilities.append(affected_vulnerability) - print("affected_vulnerability = {}".format(affected_vulnerability)) - print("affected_vulnerabilities = {}".format(affected_vulnerabilities)) - return affected_vulnerabilities - # ZAP: 2023-09-12 Tuesday 19:12:22. Not what we want but this does return the purl, which we can use to find all of its own vulns (if any). - # return package.purl - - # def get_affected_vulnerabilities_saved(self, package) -> dict: - # """ - # Return a mapping of vulnerabilities that affects the given `package`. - # """ - # # return self.get_vulnerabilities_for_a_package(package=package, fix=False) - # return self.get_vulnerabilities_for_a_package(package=self, fix=False) - purl = serializers.CharField(source="package_url") class Meta: model = Package - fields = ["url", "purl", "is_vulnerable", "affected_by_vulnerabilities"] + fields = ["url", "purl", "affected_by_vulnerabilities"] class MinimalVulnerabilitySerializer(serializers.HyperlinkedModelSerializer): @@ -167,41 +109,9 @@ def to_representation(self, instance): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") - # ZAP: 2023-09-14 Thursday 11:34:26. Add closest fixed package from the Package UI here. - - closest_fixed_package = serializers.SerializerMethodField("get_closest_fixed_package") - - def get_closest_fixed_package(self, instance): - return "Coming soon!" - - # test_data = super().to_representation(instance) - # vulnerabilities = [vuln["vulnerability"] for vuln in test_data["vulnerabilities"]] - # test_data["vulnerabilities"] = vulnerabilities - - # return test_data - - # # ZAP: From the PackageSerializer class: - # # next_non_vulnerable_version = serializers.SerializerMethodField("get_closest_non_vulnerable") - - # def get_closest_non_vulnerable(self, package): - # closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) - # if closest_non_vulnerable: - # return closest_non_vulnerable.version - # else: - # return None - class Meta: model = Vulnerability - # fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] - fields = [ - "url", - "vulnerability_id", - "summary", - "references", - "fixed_packages", - "closest_fixed_package", - "aliases", - ] + fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer): @@ -214,34 +124,8 @@ class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") - # ALERT: 2023-09-14 Thursday 15:16:05. For some reason, favorite_color is not appearing in the output. - # favorite_color = serializers.SerializerMethodField("get_favorite_color") - - # # def get_closest_non_vulnerable(self, package): - # def get_favorite_color(self, package): - # # closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) - # # if closest_non_vulnerable: - # # return closest_non_vulnerable.version - # # else: - # # return None - # return "red!" - - favorite_color = MinimalPackageSerializer( - many=True, source="filtered_fixed_packages", read_only=True - ) - class Meta: model = Vulnerability - # fields = [ - # "url", - # "vulnerability_id", - # "summary", - # "aliases", - # "fixed_packages", - # "affected_packages", - # "references", - # ] - fields = [ "url", "vulnerability_id", @@ -250,7 +134,6 @@ class Meta: "fixed_packages", "affected_packages", "references", - "favorite_color", ] @@ -259,12 +142,12 @@ class PackageSerializer(serializers.HyperlinkedModelSerializer): Lookup software package using Package URLs """ - next_non_vulnerable_version = serializers.SerializerMethodField("get_closest_non_vulnerable") + next_non_vulnerable_version = serializers.SerializerMethodField("get_next_non_vulnerable") - def get_closest_non_vulnerable(self, package): - closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) - if closest_non_vulnerable: - return closest_non_vulnerable.version + def get_next_non_vulnerable(self, package): + next_non_vulnerable = package.fixed_package_details.get("next_non_vulnerable", None) + if next_non_vulnerable: + return next_non_vulnerable.version else: return None @@ -285,7 +168,7 @@ def get_latest_non_vulnerable(self, package): def get_fixed_packages(self, package): """ - Return a queryset of all packages that fixes a vulnerability with + Return a queryset of all packages that fix a vulnerability with same type, namespace, name, subpath and qualifiers of the `package` """ return Package.objects.filter( @@ -300,7 +183,7 @@ def get_fixed_packages(self, package): def get_vulnerabilities_for_a_package(self, package, fix) -> dict: """ Return a mapping of vulnerabilities data related to the given `package`. - Return vulnerabilities that affects the `package` if given `fix` flag is False, + Return vulnerabilities that affect the `package` if given `fix` flag is False, otherwise return vulnerabilities fixed by the `package`. """ fixed_packages = self.get_fixed_packages(package=package) @@ -312,7 +195,6 @@ def get_vulnerabilities_for_a_package(self, package, fix) -> dict: to_attr="filtered_fixed_packages", ) ) - # so this calls VulnSerializerRefsAndSummary return VulnSerializerRefsAndSummary( instance=qs, many=True, @@ -342,10 +224,10 @@ class Meta: "version", "qualifiers", "subpath", - "affected_by_vulnerabilities", - "fixing_vulnerabilities", "next_non_vulnerable_version", "latest_non_vulnerable_version", + "affected_by_vulnerabilities", + "fixing_vulnerabilities", ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 8a3436436..b2630330e 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -673,14 +673,14 @@ def current_version(self): @property def fixed_package_details(self): """ - Return a mapping of vulnerabilities that affect this package and the closest and + Return a mapping of vulnerabilities that affect this package and the next and latest non-vulnerable versions. """ package_details = {} package_details["purl"] = PackageURL.from_string(self.purl) - closest_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() - package_details["closest_non_vulnerable"] = closest_non_vulnerable + next_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() + package_details["next_non_vulnerable"] = next_non_vulnerable package_details["latest_non_vulnerable"] = latest_non_vulnerable package_details["vulnerabilities"] = self.get_affecting_vulnerabilities() @@ -689,7 +689,7 @@ def fixed_package_details(self): def get_non_vulnerable_versions(self): """ - Return a tuple of the closest and latest non-vulnerable versions as PackageURLs. Return a tuple of + Return a tuple of the next and latest non-vulnerable versions as PackageURLs. Return a tuple of (None, None) if there is no non-vulnerable version. """ package_versions = self.get_fixed_by_package_versions(fix=False) @@ -706,13 +706,13 @@ def get_non_vulnerable_versions(self): if later_non_vulnerable_versions: sorted_versions = self.sort_by_version(later_non_vulnerable_versions) - closest_non_vulnerable_version = sorted_versions[0] + next_non_vulnerable_version = sorted_versions[0] latest_non_vulnerable_version = sorted_versions[-1] - closest_non_vulnerable = PackageURL.from_string(closest_non_vulnerable_version.purl) + next_non_vulnerable = PackageURL.from_string(next_non_vulnerable_version.purl) latest_non_vulnerable = PackageURL.from_string(latest_non_vulnerable_version.purl) - return closest_non_vulnerable, latest_non_vulnerable + return next_non_vulnerable, latest_non_vulnerable return None, None @@ -746,23 +746,33 @@ def get_affecting_vulnerabilities(self): if fixed_version > self.current_version: later_fixed_packages.append(fixed_pkg) - closest_fixed_package = None - closest_fixed_package_vulns = [] + next_fixed_package = None + next_fixed_package_vulns = [] + sort_fixed_by_packages_by_version = [] if later_fixed_packages: sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) - closest_fixed_package = sort_fixed_by_packages_by_version[0] - fixed_by_purl = PackageURL.from_string(closest_fixed_package.purl) - closest_fixed_package_vulns = list(closest_fixed_package.affected_by) + + fixed_by_pkgs = [] for vuln_details in package_details_vulns: if vuln_details["vulnerability"] != vuln: continue - vuln_details["fixed_by_purl"] = None + vuln_details["fixed_by_purl"] = [] vuln_details["fixed_by_purl_vulnerabilities"] = [] - if closest_fixed_package: - vuln_details["fixed_by_purl"] = fixed_by_purl - vuln_details["fixed_by_purl_vulnerabilities"] = closest_fixed_package_vulns + + for fixed_by_pkg in sort_fixed_by_packages_by_version: + fixed_by_package_details = {} + fixed_by_purl = PackageURL.from_string(fixed_by_pkg.purl) + next_fixed_package_vulns = list(fixed_by_pkg.affected_by) + + fixed_by_package_details["fixed_by_purl"] = fixed_by_purl + fixed_by_package_details[ + "fixed_by_purl_vulnerabilities" + ] = next_fixed_package_vulns + fixed_by_pkgs.append(fixed_by_package_details) + + vuln_details["fixed_by_package_details"] = fixed_by_pkgs return package_details_vulns diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 246b3733e..c70adaee0 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -50,8 +50,8 @@ Next non-vulnerable version - + @@ -116,45 +116,49 @@ {% if key == "vulnerabilities" %} {% for vuln in value %} {% if vuln.vulnerability.vulnerability_id == vulnerability.vulnerability_id %} - {% if vuln.fixed_by_purl is None %} - There are no reported fixed versions. + {% if vuln.fixed_by_package_details is None %} + There are no reported fixed by versions. {% else %} - {% if vuln.fixed_by_purl_vulnerabilities|length == 0 %} - {{ vuln.fixed_by_purl.version }} -
    - Affected by 0 other vulnerabilities. - {% else %} - {{ vuln.fixed_by_purl.version }} - {% if vuln.fixed_by_purl_vulnerabilities|length != 1 %} -
    - Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + {% for fixed_pkg in vuln.fixed_by_package_details %} +
    + {% if fixed_pkg.fixed_by_purl_vulnerabilities|length == 0 %} + {{ fixed_pkg.fixed_by_purl.version }} +
    + Affected by 0 other vulnerabilities. {% else %} -
    - Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerability. - {% endif %} + {{ fixed_pkg.fixed_by_purl.version }} + {% if fixed_pkg.fixed_by_purl_vulnerabilities|length != 1 %} +
    + Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + {% else %} +
    + Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other vulnerability. + {% endif %} -
    -
    - -
    -
    - There are no known affected packages. + There are no known affected packages.
    - See Affected packages tab for more + See Affected packages tab for more
    - {% if fixed_package_details.closest_non_vulnerable.version %} - {{ fixed_package_details.closest_non_vulnerable.version }} + {% if fixed_package_details.next_non_vulnerable.version %} + {{ fixed_package_details.next_non_vulnerable.version }} {% else %} None. {% endif %} @@ -85,7 +85,7 @@
    Vulnerability SummaryFixed byFixed by
    +
    + Let's try to display fixing packages for this package: {{ package.purl }} +
    + package.purl = {{ package.purl }} +
    +
    + package.qualifiers = {{ package.qualifiers }} +
    +
    + package.vulnerabilities = {{ package.vulnerabilities }} +
    +
    + package.package_url = {{ package.package_url }} +
    +
    + package.plain_package_url = {{ package.plain_package_url }} +
    +
    + package.purl_object = {{ package.purl_object }} +
    + +
    + package.fixing = {{ package.fixing }} +
    +
    + package.fixed_packages = Server Error (500) +
    +
    + package.is_vulnerable = {{ package.is_vulnerable }} +
    +
    + package.get_absolute_url = {{ package.get_absolute_url }} +
    + +
    + package.affected_by = {{ package.affected_by }} +
    +
    + package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} +
    + + {% for abc in package.affected_by %} + +
    {{ abc }} -- {{ abc.fixed_by_packages }}
    +
    {{ abc }} -- {% for pkg in abc.fixed_by_packages %}{{ pkg.purl }}{% endfor %}
    + + {% endfor %} + + + +
    +
    Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }}) @@ -51,6 +103,7 @@ Vulnerability Summary Aliases + Test fixing PURLs @@ -74,10 +127,18 @@ {% endif %} {% endfor %} + + + + {% for pkg in vulnerability.fixed_by_packages %} + {{ pkg.purl }} + {% endfor %} + {% empty %} - + + This package is not known to be affected by vulnerabilities. diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index fe406a0a8..8ca39d77c 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -186,7 +186,6 @@ class ApiUserCreateView(generic.CreateView): template_name = "api_user_creation_form.html" def form_valid(self, form): - try: response = super().form_valid(form) except ValidationError: From dff950f3e9fca08b45d19b07d678fc898a858df9 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Fri, 28 Jul 2023 09:26:40 -0700 Subject: [PATCH 22/53] Explore context and Package class approaches for affected-fixed package matching #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Note that my updated code is still in testing/dev stage and has not yet been completed or cleaned. Signed-off-by: John M. Horan --- vulnerabilities/models.py | 80 +++++++++-- .../templates/package_details.html | 134 +++++++++++++++--- vulnerabilities/views.py | 35 +++++ 3 files changed, 221 insertions(+), 28 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index fcd3a0cc9..97ac090e6 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -23,6 +23,7 @@ from django.core.validators import MinValueValidator from django.db import models from django.db.models import Count +from django.db.models import Prefetch from django.db.models import Q from django.db.models.functions import Length from django.db.models.functions import Trim @@ -611,18 +612,6 @@ def affected_by(self): # legacy aliases vulnerable_to = affected_by - @property - def test_get_fixing_purls(self): - """ - This is a test -- the goal is to display the closest fixing version for a PURL that is greater - than the affected version and is the same type. We want to filter on type, namespace, - name, qualifiers and subpath for the affected PURL. - """ - return [ - abc.fixed_by_packages - for abc in self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - ] - @property # TODO: consider renaming to "fixes" or "fixing" ? (TBD) and updating the docstring def fixing(self): @@ -654,6 +643,73 @@ def get_absolute_url(self): """ return reverse("package_details", args=[self.purl]) + def get_fixed_packages(self, package): + """ + Return a queryset of all packages that fix a vulnerability with + same type, namespace, name, subpath and qualifiers of the `package` + """ + return Package.objects.filter( + name=package.name, + namespace=package.namespace, + type=package.type, + qualifiers=package.qualifiers, + subpath=package.subpath, + packagerelatedvulnerability__fix=True, + ).distinct() + + @property + def test_get_fixing_purls(self): + """ + This is a test -- the goal is to display the closest fixing version for a PURL that is greater + than the affected version and is the same type. We want to filter on type, namespace, + name, qualifiers and subpath for the affected PURL. + """ + + fixed_packages = self.get_fixed_packages(package=self) + + # Prefetch: + + # Where do we get "fix" from? + # qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=fix) + + # Not clear to me how we use this: + # qs = qs.prefetch_related( + # Prefetch( + # "packages", + # queryset=fixed_packages, + # to_attr="filtered_fixed_packages", + # ) + # ) + + matching_fixed_packages = [] + + # closest_subsequent_fixed_package = [] + + test_dict = {"affected_purl": self.purl} + test_dict["vulnerabilities"] = [] + + for vuln in self.affected_by: + test_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + + for fixing_pkg in vuln.fixed_by_packages: + # Do not add "backports". TODO: Do we need to use univers to compare versions? + if fixing_pkg in fixed_packages and fixing_pkg.version > self.version: + matching_fixed_packages.append(fixing_pkg) + + # TODO: We also need to check if there is more than one fixing package and if so, keep only the package closest to the affect package's version. + # TODO: Do we want to check whether the fixing version has any vulnerabilities of its own? + + # ======================================================== + print("\ntest_dict = \n{}\n".format(test_dict)) + print(json.dumps(test_dict, indent=4)) + # ======================================================== + + return matching_fixed_packages + + +# 2023-07-27 Thursday 09:50:45. JMH: check this out re related api.py code +# TODO: Not clear how this is involved. + class PackageRelatedVulnerability(models.Model): """ diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 5cc93fcb0..b62722c05 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -43,52 +43,131 @@
    Let's try to display fixing packages for this package: {{ package.purl }}
    - package.purl = {{ package.purl }} + package.purl = {{ package.purl }}
    - package.qualifiers = {{ package.qualifiers }} + package.qualifiers = {{ package.qualifiers }}
    - package.vulnerabilities = {{ package.vulnerabilities }} + package.vulnerabilities = {{ package.vulnerabilities }}
    - package.package_url = {{ package.package_url }} + package.package_url = {{ package.package_url }}
    - package.plain_package_url = {{ package.plain_package_url }} + package.plain_package_url = {{ package.plain_package_url }}
    - package.purl_object = {{ package.purl_object }} + package.purl_object = {{ package.purl_object }}
    - package.fixing = {{ package.fixing }} + package.fixing = {{ package.fixing }}
    - package.fixed_packages = Server Error (500) + package.fixed_packages = Server Error (500)
    - package.is_vulnerable = {{ package.is_vulnerable }} + package.is_vulnerable = {{ package.is_vulnerable }}
    - package.get_absolute_url = {{ package.get_absolute_url }} + package.get_absolute_url = {{ package.get_absolute_url }}
    - package.affected_by = {{ package.affected_by }} + package.affected_by = {{ package.affected_by }}
    + +
    + package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} +
    +
    - package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} + package.test_get_fixing_purls = {{ package.test_get_fixing_purls }}
    {% for abc in package.affected_by %} -
    {{ abc }} -- {{ abc.fixed_by_packages }}
    -
    {{ abc }} -- {% for pkg in abc.fixed_by_packages %}{{ pkg.purl }}{% endfor %}
    +
    {{ abc }} -- {{ abc.fixed_by_packages }}
    +
    {{ abc }} -- {% for pkg in abc.fixed_by_packages %}{{ pkg.purl }}{% endfor %}
    + + {% endfor %} + +
    abc.test_matching_fixed_by_packages
    + {% for abc in package.affected_by %} + + + +
    {{ abc }} -- + {% for pkg in abc.test_matching_fixed_by_packages %} +
    {{ pkg.purl }}
    + {% endfor %} +
    {% endfor %} +
    +
    test01
    + {% for thing in test01 %} + {{ thing }} +
    + {% endfor %} +
    + +
    +
    test02
    + {% for thing in test02 %} + - {{ thing }} +
    + {% endfor %} +
    + +
    +
    test03
    + {% for thing in test03 %} + - {{ thing }} +
    + {% for stuff in thing.test_matching_fixed_by_packages %} + ==> {{ stuff }} +
    + {% endfor %} +
    + {% endfor %} +
    + +
    +
    test04
    + {% for thing in test04 %} + - {{ thing }} +
    + {% endfor %} +
    + +
    + +
    context["test_get_fixing_purls"]
    + {% for thing in test_get_fixing_purls %} + - {{ thing }} +
    + {% endfor %} + {% for vulnerability in affected_by_vulnerabilities %} +
    + {{ vulnerability }} +
    +
    + {% for pkg in vulnerability.fixed_by_packages %} + {% if pkg in test_get_fixing_purls %} +
    + {{ pkg.purl }} +
    + {% endif %} + {% endfor %} +
    + {% endfor %} +
    @@ -103,7 +182,7 @@ Vulnerability Summary Aliases - Test fixing PURLs + Fixing Packages @@ -131,8 +210,31 @@ {% for pkg in vulnerability.fixed_by_packages %} - {{ pkg.purl }} +
    + {{ pkg.purl }} +
    + {% endfor %} + +
    + + {% for pkg in vulnerability.fixed_by_packages %} + {% if pkg in test04 %} +
    + {{ pkg.purl }} +
    + {% endif %} + {% endfor %} + +
    + + {% for pkg in vulnerability.fixed_by_packages %} + {% if pkg in test_get_fixing_purls %} +
    + {{ pkg.purl }} +
    + {% endif %} {% endfor %} + {% empty %} diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 8ca39d77c..b90e5c306 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -84,6 +84,41 @@ def get_context_data(self, **kwargs): context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) + context["test01"] = [ + "aaa", + "bbb", + "ccc", + ] + context["test02"] = [ + package, + package.type, + package.namespace, + package.name, + package.version, + package.qualifiers, + package.subpath, + ] + context["test03"] = package.affected_by + + # ======================================================== + context["test04"] = [ + fixing_pkg + for vuln in package.affected_by + # for fixing_pkg in vuln.test_matching_fixed_by_packages + for fixing_pkg in vuln.fixed_by_packages + if fixing_pkg.type == package.type + and fixing_pkg.namespace == package.namespace + and fixing_pkg.name == package.name + and fixing_pkg.qualifiers == package.qualifiers + and fixing_pkg.subpath == package.subpath + # I think this version comparison requires the use of univers. + # Plus we just need the closest one, not all that are greater than. + and fixing_pkg.version > package.version + ] + # ======================================================== + context["test_get_fixing_purls"] = package.test_get_fixing_purls + # ======================================================== + return context def get_object(self, queryset=None): From 627d117ac547307fb33b0fb66dcaf1c079614379 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Sat, 29 Jul 2023 18:00:26 -0700 Subject: [PATCH 23/53] Add Prefetch and univers-based version comparison #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 115 +++++++++---- .../templates/package_details.html | 155 +----------------- vulnerabilities/views.py | 35 +--- 3 files changed, 86 insertions(+), 219 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 97ac090e6..dbfb01fc9 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -33,6 +33,7 @@ from packageurl.contrib.django.models import PackageURLQuerySet from packageurl.contrib.django.models import without_empty_values from rest_framework.authtoken.models import Token +from univers import versions from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import build_vcid @@ -657,58 +658,100 @@ def get_fixed_packages(self, package): packagerelatedvulnerability__fix=True, ).distinct() - @property - def test_get_fixing_purls(self): + def assign_univers_version(self, fixing_pkg): """ - This is a test -- the goal is to display the closest fixing version for a PURL that is greater - than the affected version and is the same type. We want to filter on type, namespace, - name, qualifiers and subpath for the affected PURL. + Identify which univers version applies to the two packages to be compared (self and a fixing package), + evaluate whether the fixing_pkg version is > than the target affected package, and + return True or False. """ - fixed_packages = self.get_fixed_packages(package=self) + # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. - # Prefetch: + # Many more to be added. + match_type_to_univers_version = { + "conan": versions.ConanVersion, + "maven": versions.MavenVersion, + } - # Where do we get "fix" from? - # qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=fix) + command_name = "" - # Not clear to me how we use this: - # qs = qs.prefetch_related( - # Prefetch( - # "packages", - # queryset=fixed_packages, - # to_attr="filtered_fixed_packages", - # ) - # ) + type_to_version = match_type_to_univers_version.get(fixing_pkg.type) + if type_to_version: + print("\t--------------------------") + print("\ttype_to_version = {}\n".format(type_to_version)) + command_name = type_to_version - matching_fixed_packages = [] + else: + print("\ttype_to_version = NO MATCH\n") + command_name = versions.Version - # closest_subsequent_fixed_package = [] + if command_name(fixing_pkg.version) > command_name(self.version): + return True + else: + return False - test_dict = {"affected_purl": self.purl} - test_dict["vulnerabilities"] = [] + @property + def get_fixing_packages(self): + """ + This function identifies the closest fixing version that is greater than the affected version and + is the same type, namespace, name, qualifiers and subpath as the affected package. + """ - for vuln in self.affected_by: - test_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + print("\nself = {}\n".format(self)) - for fixing_pkg in vuln.fixed_by_packages: - # Do not add "backports". TODO: Do we need to use univers to compare versions? - if fixing_pkg in fixed_packages and fixing_pkg.version > self.version: - matching_fixed_packages.append(fixing_pkg) + # This returns all fixing packages that match the target package (type etc.), regardless of fixed vuln. + fixed_packages = self.get_fixed_packages(package=self) - # TODO: We also need to check if there is more than one fixing package and if so, keep only the package closest to the affect package's version. - # TODO: Do we want to check whether the fixing version has any vulnerabilities of its own? + # This returns a list of the vulnerabilities that affect this package (i.e., self). + qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - # ======================================================== - print("\ntest_dict = \n{}\n".format(test_dict)) - print(json.dumps(test_dict, indent=4)) - # ======================================================== + # This takes the list of vulns affecting the current package, retrieves a list of the fixing packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages`. + # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + qs = qs.prefetch_related( + Prefetch( + "packages", + queryset=fixed_packages, + to_attr="filtered_fixed_packages", + ) + ) - return matching_fixed_packages + # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). + print("qs = {}\n".format(qs)) + prefetch_fixed_packages = [] -# 2023-07-27 Thursday 09:50:45. JMH: check this out re related api.py code -# TODO: Not clear how this is involved. + vuln_count = 0 + for vuln in qs: + print("vuln = {}\n".format(vuln)) + print( + "\tqs[vuln_count].filtered_fixed_packages = {}".format( + qs[vuln_count].filtered_fixed_packages + ) + ) + print("") + + # Check the Prefetch qs. + # TODO: Do we want to check whether the fixing version has any vulnerabilities of its own? + for fixing_pkg in qs[vuln_count].filtered_fixed_packages: + print("\tfixing_pkg = {}".format(fixing_pkg)) + print("\tfixing_pkg.type = {}".format(fixing_pkg.type)) + print("\tfixing_pkg.version = {}".format(fixing_pkg.version)) + print("\t--------------------------") + print("\tself.type = {}".format(self.type)) + print("\tself.version = {}".format(self.version)) + + # Assign univers version and compare: False = fixing_pkg.version < self.version (affected version). + # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. + immediate_fix = self.assign_univers_version(fixing_pkg) + print("\t--------------------------") + print("\timmediate_fix = {}\n".format(immediate_fix)) + + if fixing_pkg in fixed_packages and immediate_fix: + prefetch_fixed_packages.append(fixing_pkg) + + vuln_count += 1 + + return prefetch_fixed_packages class PackageRelatedVulnerability(models.Model): diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index b62722c05..e07ae199e 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -40,138 +40,8 @@
    -
    - Let's try to display fixing packages for this package: {{ package.purl }} -
    - package.purl = {{ package.purl }} -
    -
    - package.qualifiers = {{ package.qualifiers }} -
    -
    - package.vulnerabilities = {{ package.vulnerabilities }} -
    -
    - package.package_url = {{ package.package_url }} -
    -
    - package.plain_package_url = {{ package.plain_package_url }} -
    -
    - package.purl_object = {{ package.purl_object }} -
    - -
    - package.fixing = {{ package.fixing }} -
    -
    - package.fixed_packages = Server Error (500) -
    -
    - package.is_vulnerable = {{ package.is_vulnerable }} -
    -
    - package.get_absolute_url = {{ package.get_absolute_url }} -
    - -
    - package.affected_by = {{ package.affected_by }} -
    - -
    - package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} -
    - -
    - package.test_get_fixing_purls = {{ package.test_get_fixing_purls }} -
    - - {% for abc in package.affected_by %} - -
    {{ abc }} -- {{ abc.fixed_by_packages }}
    -
    {{ abc }} -- {% for pkg in abc.fixed_by_packages %}{{ pkg.purl }}{% endfor %}
    - - {% endfor %} - -
    abc.test_matching_fixed_by_packages
    - {% for abc in package.affected_by %} - - - -
    {{ abc }} -- - {% for pkg in abc.test_matching_fixed_by_packages %} -
    {{ pkg.purl }}
    - {% endfor %} -
    - - {% endfor %} - -
    -
    test01
    - {% for thing in test01 %} - {{ thing }} -
    - {% endfor %} -
    - -
    -
    test02
    - {% for thing in test02 %} - - {{ thing }} -
    - {% endfor %} -
    - -
    -
    test03
    - {% for thing in test03 %} - - {{ thing }} -
    - {% for stuff in thing.test_matching_fixed_by_packages %} - ==> {{ stuff }} -
    - {% endfor %} -
    - {% endfor %} -
    - -
    -
    test04
    - {% for thing in test04 %} - - {{ thing }} -
    - {% endfor %} -
    - -
    - -
    context["test_get_fixing_purls"]
    - {% for thing in test_get_fixing_purls %} - - {{ thing }} -
    - {% endfor %} - - {% for vulnerability in affected_by_vulnerabilities %} -
    - {{ vulnerability }} -
    -
    - {% for pkg in vulnerability.fixed_by_packages %} - {% if pkg in test_get_fixing_purls %} -
    - {{ pkg.purl }} -
    - {% endif %} - {% endfor %} -
    - {% endfor %} -
    - -
    -
    +
    During testing, red = all packages that fix the vulnerability (for pkg in vulnerability.fixed_by_packages) and green = the closest fixing package whose version is greater than the affected package's version (if pkg in get_fixing_packages).
    Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }})
    @@ -182,7 +52,7 @@ Vulnerability Summary Aliases - Fixing Packages + Fixing Packages @@ -207,39 +77,26 @@ {% endfor %} - - {% for pkg in vulnerability.fixed_by_packages %} -
    +
    {{ pkg.purl }}
    {% endfor %} -
    +
    {% for pkg in vulnerability.fixed_by_packages %} - {% if pkg in test04 %} -
    + {% if pkg in get_fixing_packages %} +
    {{ pkg.purl }}
    {% endif %} {% endfor %} -
    - - {% for pkg in vulnerability.fixed_by_packages %} - {% if pkg in test_get_fixing_purls %} -
    - {{ pkg.purl }} -
    - {% endif %} - {% endfor %} - {% empty %} - This package is not known to be affected by vulnerabilities. diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index b90e5c306..1030fb410 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -84,40 +84,7 @@ def get_context_data(self, **kwargs): context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) - context["test01"] = [ - "aaa", - "bbb", - "ccc", - ] - context["test02"] = [ - package, - package.type, - package.namespace, - package.name, - package.version, - package.qualifiers, - package.subpath, - ] - context["test03"] = package.affected_by - - # ======================================================== - context["test04"] = [ - fixing_pkg - for vuln in package.affected_by - # for fixing_pkg in vuln.test_matching_fixed_by_packages - for fixing_pkg in vuln.fixed_by_packages - if fixing_pkg.type == package.type - and fixing_pkg.namespace == package.namespace - and fixing_pkg.name == package.name - and fixing_pkg.qualifiers == package.qualifiers - and fixing_pkg.subpath == package.subpath - # I think this version comparison requires the use of univers. - # Plus we just need the closest one, not all that are greater than. - and fixing_pkg.version > package.version - ] - # ======================================================== - context["test_get_fixing_purls"] = package.test_get_fixing_purls - # ======================================================== + context["get_fixing_packages"] = package.get_fixing_packages return context From 65d82a7c3f314c94594f03c6b9be0a2ea14c5347 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 7 Aug 2023 19:04:39 -0700 Subject: [PATCH 24/53] Update affected-fixed package matching #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 663 ++++++++++++++++-- .../templates/package_details.html | 140 +++- vulnerabilities/tests/test_models.py | 65 ++ vulnerabilities/views.py | 5 +- vulnerablecode/settings.py | 4 +- vulnerablecode/static/css/custom.css | 99 +++ 6 files changed, 927 insertions(+), 49 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index dbfb01fc9..55f2b00e3 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -658,100 +658,685 @@ def get_fixed_packages(self, package): packagerelatedvulnerability__fix=True, ).distinct() - def assign_univers_version(self, fixing_pkg): + def get_sibling_packages(self, package): """ - Identify which univers version applies to the two packages to be compared (self and a fixing package), - evaluate whether the fixing_pkg version is > than the target affected package, and + Return a queryset of all packages with the same type, namespace, name, subpath and qualifiers of the `package`, whether or not they fix any vulnerability + """ + return Package.objects.filter( + name=package.name, + namespace=package.namespace, + type=package.type, + qualifiers=package.qualifiers, + subpath=package.subpath, + # packagerelatedvulnerability__fix=True, + ).distinct() + + # def assign_and_compare_univers_versions(self, fixed_pkg): + def assign_univers_version(self, fixed_pkg): + """ + Identify which univers version applies to the two packages to be compared (self and a fixed package), + evaluate whether the fixed_pkg version is > than the target affected package, and return True or False. """ - # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. + # TODO: Instead of return True or False based on evaluating the incoming fixed_pkg type to a univers version and then checking whether the fixed version is greater than the affected (self) version, we'll just use the incoming type to assign and return a univers version -- as command_name -- to be used in get_closest_fixed_package() below to add all greater-than versions to the later_matching_fixed_packages list -- which in turn will be fed to self.sort_by_version(later_matching_fixed_packages) # Many more to be added. match_type_to_univers_version = { "conan": versions.ConanVersion, + "deb": versions.DebianVersion, "maven": versions.MavenVersion, + "openssl": versions.OpensslVersion, + "pypi": versions.PypiVersion, } command_name = "" - type_to_version = match_type_to_univers_version.get(fixing_pkg.type) - if type_to_version: + matched_type_to_version = match_type_to_univers_version.get(fixed_pkg.type) + if matched_type_to_version: print("\t--------------------------") - print("\ttype_to_version = {}\n".format(type_to_version)) - command_name = type_to_version + print("*** matched_type_to_version = {}".format(matched_type_to_version)) + command_name = matched_type_to_version else: - print("\ttype_to_version = NO MATCH\n") - command_name = versions.Version + print("\t--------------------------") + print("*** matched_type_to_version = NO MATCH") + # Using "command_name = versions.Version", the test + # assert versions.Version("0.9") < versions.Version("0.10") + # fails! + # command_name = versions.Version + # Use this as a default fallback instead. + command_name = versions.SemverVersion + + # if command_name(fixed_pkg.version) > command_name(self.version): + # return True + # else: + # return False + + # Instead return command_name for recipient to use as needed for sorting or perhaps other uses + return command_name + + def sort_by_version(self, later_matching_fixed_packages): + # Incoming is a list of + + # ALERT: added this to address server error 500 but related error arose: line 908, in get_closest_fixed_package + # HOT: How is this related to the source of the server error (500)? + if len(later_matching_fixed_packages) == 0: + return + + # Replace find_closest_fixed_by_package()? + print("\nlater_matching_fixed_packages = {}".format(later_matching_fixed_packages)) + print("\nlater_matching_fixed_packages[0] = {}".format(later_matching_fixed_packages[0])) + print( + "\ntype(later_matching_fixed_packages[0]) = {}".format( + type(later_matching_fixed_packages[0]) + ) + ) + # NOTE: This gives us the PURL type but instead we want the PURL itself to pass to assign_univers_version(self, fixed_pkg) and get the command_name in return, which we'll then use in the sort process. + print( + "\nlater_matching_fixed_packages[0].type = {}".format( + later_matching_fixed_packages[0].type + ) + ) + print( + "\ntype(later_matching_fixed_packages[0].type) = {}".format( + type(later_matching_fixed_packages[0].type) + ) + ) - if command_name(fixing_pkg.version) > command_name(self.version): - return True - else: - return False + # Incoming is a list -- later_matching_fixed_packages + + # We'll use assign_univers_version() above to get the univers version as a command_name. + # But what do we pass to it? The [0] index of the incoming list, i.e., later_matching_fixed_packages[0]? + command_name = self.assign_univers_version(later_matching_fixed_packages[0]) + + print("\n>>> command_name = {}\n".format(command_name)) + + # TODO: Maybe we don't need to convert to a PURL, a list of dictionaries etc.?? + print( + "\n+++++++ later_matching_fixed_packages[0].version = {}".format( + later_matching_fixed_packages[0].version + ) + ) + + # sort + test_sort_by_version = [] + test_sort_by_version = sorted( + # later_matching_fixed_packages, key=lambda x: versions.DebianVersion(x["version"]) + later_matching_fixed_packages, + # key=lambda x: versions.MavenVersion(x.version), + key=lambda x: command_name(x.version), + ) + + print("\ntest_sort_by_version = {}\n".format(test_sort_by_version)) + + return test_sort_by_version + + # convert_to_dict_list = [] + + # sorted_later_matching_fixed_packages = [] + + # # TODO: First, convert to a list of dictionaries. + # for pkg in later_matching_fixed_packages: + # # pkg is a + # print("pkg = {}".format(pkg)) + # print("type(pkg) = {}".format(type(pkg))) + + # # pkg_str is a string + # pkg_str = pkg.package_url + # print("pkg_str = {}".format(pkg_str)) + # print("type(pkg_str) = {}".format(type(pkg_str))) + + # # purl is a + # purl = PackageURL.from_string(pkg_str) + # print("purl = {}".format(purl)) + # print("type(purl) = {}".format(type(purl))) + + # purl_dict = purl.to_dict() + # print("purl_dict = {}".format(purl_dict)) + # print("type(purl_dict) = {}".format(type(purl_dict))) + + # convert_to_dict_list.append(purl.to_dict()) + # print("HELLO\n") + + # print("\nconvert_to_dict_list = {}\n".format(convert_to_dict_list)) + + # return convert_to_dict_list + # ========================================================== + # sorted_later_matching_fixed_packages = sorted( + # later_matching_fixed_packages, key=lambda x: versions.MavenVersion(x["version"]) + # ) + # print( + # "\nsorted_later_matching_fixed_packages = {}\n".format( + # sorted_later_matching_fixed_packages + # ) + # ) + # print("\n".join(map(str, sorted_later_matching_fixed_packages))) + + # return what? + + # ========================================================== + # ========================================================== + + # def find_closest_fixed_by_package(self, later_matching_fixed_packages): + # # Maybe use sort_by_version() above instead? + # # take the incoming list later_matching_fixed_packages, convert to list of dictionaries, sort by version using univers.version.[version class], choose the top i.e., index [0] and convert back to PURL and return that PURL. + # print("\nlater_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) + + # closest_fixed_by_package = "TBD" + + # return closest_fixed_by_package @property - def get_fixing_packages(self): + # def get_fixing_packages(self): + def get_closest_fixed_package(self): """ - This function identifies the closest fixing version that is greater than the affected version and + This function identifies the closest fixed package version that is greater than the affected package version and is the same type, namespace, name, qualifiers and subpath as the affected package. """ print("\nself = {}\n".format(self)) - # This returns all fixing packages that match the target package (type etc.), regardless of fixed vuln. - fixed_packages = self.get_fixed_packages(package=self) + # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. + # fixed_packages = self.get_fixed_packages(package=self) + # This is clearer. + matching_fixed_packages = self.get_fixed_packages(package=self) # This returns a list of the vulnerabilities that affect this package (i.e., self). qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - # This takes the list of vulns affecting the current package, retrieves a list of the fixing packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages`. - # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). + # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). qs = qs.prefetch_related( Prefetch( "packages", - queryset=fixed_packages, - to_attr="filtered_fixed_packages", + # queryset=fixed_packages, + queryset=matching_fixed_packages, + # to_attr="filtered_fixed_packages", + to_attr="matching_fixed_packages", ) ) # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). print("qs = {}\n".format(qs)) - prefetch_fixed_packages = [] + # ************************************************************************ + + later_matching_fixed_packages = [] vuln_count = 0 for vuln in qs: print("vuln = {}\n".format(vuln)) + # print( + # "\tqs[vuln_count].filtered_fixed_packages = {}".format( + # qs[vuln_count].filtered_fixed_packages + # ) + # ) print( - "\tqs[vuln_count].filtered_fixed_packages = {}".format( - qs[vuln_count].filtered_fixed_packages + "\tqs[vuln_count].matching_fixed_packages = {}".format( + qs[vuln_count].matching_fixed_packages ) ) print("") # Check the Prefetch qs. - # TODO: Do we want to check whether the fixing version has any vulnerabilities of its own? - for fixing_pkg in qs[vuln_count].filtered_fixed_packages: - print("\tfixing_pkg = {}".format(fixing_pkg)) - print("\tfixing_pkg.type = {}".format(fixing_pkg.type)) - print("\tfixing_pkg.version = {}".format(fixing_pkg.version)) + # TODO: Do we want to check whether the fixed version has any vulnerabilities of its own? + # for fixed_pkg in qs[vuln_count].filtered_fixed_packages: + for fixed_pkg in qs[vuln_count].matching_fixed_packages: + print("\tfixed_pkg = {}".format(fixed_pkg)) + print("\tfixed_pkg.type = {}".format(fixed_pkg.type)) + print("\tfixed_pkg.version = {}".format(fixed_pkg.version)) print("\t--------------------------") print("\tself.type = {}".format(self.type)) print("\tself.version = {}".format(self.version)) - # Assign univers version and compare: False = fixing_pkg.version < self.version (affected version). - # TODO: We also need to find which fixing_pkg is closest to the affected version -- we don't do that yet. - immediate_fix = self.assign_univers_version(fixing_pkg) - print("\t--------------------------") - print("\timmediate_fix = {}\n".format(immediate_fix)) - - if fixing_pkg in fixed_packages and immediate_fix: - prefetch_fixed_packages.append(fixing_pkg) + # Assign univers version and compare: False = fixed_pkg.version < self.version (affected version). + + # 2023-08-02 Wednesday 16:01:35. atm immediate_fix is True or False. If instead assign_and_compare_univers_versions() returns the univers version, we could get that here and then test with this or similar right here -- enabling use of the univers version function in other places as well, like a sort_by_version function! + # if command_name(fixed_pkg.version) > command_name(self.version): + # return True + # else: + # return False + # ===================================================== + # Replace this with chunk below + # immediate_fix = self.assign_and_compare_univers_versions(fixed_pkg) + # print("\t--------------------------") + # print("\timmediate_fix = {}\n".format(immediate_fix)) + + # if fixed_pkg in fixed_packages and immediate_fix: + # later_matching_fixed_packages.append(fixed_pkg) + # ===================================================== + # command_name = self.assign_and_compare_univers_versions(fixed_pkg) + # renamed + # TODO: Move this up before the for loop -- both for loops if possible -- to reduce calls! + command_name = self.assign_univers_version(fixed_pkg) + print("\nJust requested command_name >>> {}\n".format(command_name)) + # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( + # self.version + # ): + if fixed_pkg in matching_fixed_packages and command_name( + fixed_pkg.version + ) > command_name(self.version): + later_matching_fixed_packages.append(fixed_pkg) vuln_count += 1 - return prefetch_fixed_packages + # find_closest_fixed_by_package -- from the list later_matching_fixed_packages + # closest_fixed_by_package = self.find_closest_fixed_by_package(later_matching_fixed_packages) + + # TODO: or instead use this. This will be a list sorted by univers version class, and here all we need is to grab the [0] index from that list for the closest fixed by package! So we'd return a single closest_fixed_package. + # ALERT: The sort query needs to be done separately for each vulnerability because the list of fixed by packages is likely to be different. As is, we return a single sorted list of all fixed by packages for the affected package and then pass just the [0] package -- not what we want to do! + sort_fixed_by_packages_by_version = self.sort_by_version(later_matching_fixed_packages) + print( + "\nsort_fixed_by_packages_by_version = {}\n".format(sort_fixed_by_packages_by_version) + ) + # ALERT: 2023-08-05 Saturday 23:22:08. Address server error 500? + # ALERT: 2023-08-05 Saturday 23:24:50. This actusally fixed the server error (500) and I can now even see the Packafe details page for pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1 !!! + # HOT: I need to trace back the root cause of the server error (500). I suspect it's something like a record with no fixed by packages or something else that is an empty list but which i try to measure, e.g., for a print statement, or possibly for a real if condition.. + if sort_fixed_by_packages_by_version is None: + return + + closest_fixed_package = sort_fixed_by_packages_by_version[0] + + # print("\n!!! later_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) + # print( + # "\n!!! sort_fixed_by_packages_by_version = {}\n".format( + # sort_fixed_by_packages_by_version + # ) + # ) + # print("\n!!! closest_fixed_package = {}\n".format(closest_fixed_package)) + + # rebuilt_purl_from_dict = PackageURL( + # closest_fixed_package["type"], + # closest_fixed_package["namespace"], + # closest_fixed_package["name"], + # closest_fixed_package["version"], + # closest_fixed_package["qualifiers"], + # closest_fixed_package["subpath"], + # ) + # print("\n!!! rebuilt_purl_from_dict = {}\n".format(rebuilt_purl_from_dict)) + + # return later_matching_fixed_packages + return sort_fixed_by_packages_by_version + # return [closest_fixed_package] + + # return [rebuilt_purl_from_dict] + + @property + def fixed_package_details(self): + """ + This is a test that might develop into a model-based equivalent of the loops etc. I was doing/trying to do in the Jinja2 template. I'm going to add this as a context so we can see it in the template. + """ + # return "Hello" + + # vcio_dict = { + # [ + # {"VCID-2nyb-8rwu-aaag": "PURL01"}, + # {"VCID-gqhw-ngh8-aaap": "PURL02"}, + # {"some-other-id": "PURL03"}, + # ] + # } + + print("\n==> This is from the test_property_01() property.\n") + + print("\nself = {}\n".format(self)) + + # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. + # fixed_packages = self.get_fixed_packages(package=self) + # This is clearer. + matching_fixed_packages = self.get_fixed_packages(package=self) + + # This returns a list of the vulnerabilities that affect this package (i.e., self). + qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) + + # TODO: Can we get all sibling packages so that we can then determine which have 0 vulnerabilities and of these the closest and maybe the most recent as well? + + all_sibling_packages = self.get_sibling_packages(package=self) + print("\nall_sibling_packages = {}\n".format(all_sibling_packages)) + print("\nlen(all_sibling_packages) = {}\n".format(len(all_sibling_packages))) + + non_vuln_sibs = [] + for sib in all_sibling_packages: + if sib.is_vulnerable is False: + non_vuln_sibs.append(sib) + print("\nnon_vuln_sibs = {}\n".format(non_vuln_sibs)) + print("\nlen(non_vuln_sibs) = {}\n".format(len(non_vuln_sibs))) + + # Add just the greater-than versions to a new list + command_name = self.assign_univers_version(self) + print( + "\nOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO command_name = {}\n".format( + command_name + ) + ) + later_non_vuln_sibs = [] + for non_vuln_sib in non_vuln_sibs: + if command_name(non_vuln_sib.version) > command_name(self.version): + later_non_vuln_sibs.append(non_vuln_sib) + + print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) + print("\nlen(later_non_vuln_sibs) = {}\n".format(len(later_non_vuln_sibs))) + + # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). + # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + qs = qs.prefetch_related( + Prefetch( + "packages", + # queryset=fixed_packages, + queryset=matching_fixed_packages, + # to_attr="filtered_fixed_packages", + to_attr="matching_fixed_packages", + ) + ) + + # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). + print("\nzzz qs = {}\n".format(qs)) + + purl_dict = {} + + purl_dict["purl"] = self.purl + + purl_dict.update({"vulnerabilities": []}) + + # purl_dict["vulnerabilities"].append({"fruit": "orange"}) + + for vuln in qs: + print("\nzzz vuln = {}\n".format(vuln)) + print("\nzzz type(vuln) = {}\n".format(type(vuln))) + + later_matching_fixed_packages = [] + # purl_dict[vuln.vulnerability_id] = "aaa" + # purl_dict.update({"vulnerability": vuln.vulnerability_id}) + + purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + + # TODO:2023-08-05 Saturday 13:12:28. This returns a list of matching fixed packages for this specific vuln! + vuln_matching_fixed_packages = vuln.matching_fixed_packages + print("\nzzz self.purl = {}\n".format(self.purl)) + print("\nzzz vuln = {}\n".format(vuln)) + print("\nzzz vuln_matching_fixed_packages = {}\n".format(vuln_matching_fixed_packages)) + + # TODO: So we need to sort this list by version using the correct univers version and then return the [0] index in that sorted list + # QUESTION: Do we still need to remove lesser-than fixed packages or did we already do that? + + # ============================================================= + # command_name = self.assign_univers_version(fixed_pkg) + command_name = self.assign_univers_version(self) + print("\nzzz command_name = {}\n".format(command_name)) + + # ALERT: What if there are no fixed by packages? The following thows an error because the list 'vuln_matching_fixed_packages' is empty! + # [I fixed this, right? ;-] + + closest_fixed_package = "" + + if len(vuln_matching_fixed_packages) > 0: + + for fixed_pkg in vuln_matching_fixed_packages: + if fixed_pkg in matching_fixed_packages and command_name( + fixed_pkg.version + ) > command_name(self.version): + later_matching_fixed_packages.append(fixed_pkg) + + # print("\nJust requested command_name >>> {}\n".format(command_name)) + # # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( + # # self.version + # # ): + # if fixed_pkg in matching_fixed_packages and command_name( + # fixed_pkg.version + # ) > command_name(self.version): + # later_matching_fixed_packages.append(fixed_pkg) + # ============================================================= + # later_matching_fixed_packages = vuln.matching_fixed_packages + + print( + "\nzzz later_matching_fixed_packages = {}\n".format( + later_matching_fixed_packages + ) + ) + + sort_fixed_by_packages_by_version = self.sort_by_version( + later_matching_fixed_packages + ) + print( + "\nzzz sort_fixed_by_packages_by_version = {}\n".format( + sort_fixed_by_packages_by_version + ) + ) + closest_fixed_package = sort_fixed_by_packages_by_version[0] + # closest_fixed_package = sort_fixed_by_packages_by_version[0].purl + # 2023-08-06 Sunday 11:15:03. This returns a queryset of vulns affecting this package. + # HOT: How do we get the closest fixed by package vuln count and list of vulns? I keep getting errors. ;-) + closest_fixed_package_vulns = closest_fixed_package.affected_by + # closest_fixed_package_vulns_list = list(closest_fixed_package_vulns) + # ALERT: 2023-08-06 Sunday 14:25:49. This did the trick! + + # closest_fixed_package_vulns_list = [ + # i.vulnerability_id for i in closest_fixed_package_vulns + # ] + + # 2023-08-06 Sunday 16:53:27. Try a named tuple to pass the vuln's vulnerability+id and get_absolute_url. + # FixedPackageVuln = namedtuple("FixedPackageVuln", "vuln_id, vuln_get_absolute_url") + # closest_fixed_package_vulns_list = [ + # FixedPackageVuln( + # vuln_id=fixed_pkg_vuln.vulnerability_id, + # vuln_get_absolute_url=fixed_pkg_vuln.get_absolute_url(), + # ) + # for fixed_pkg_vuln in closest_fixed_package_vulns + # ] + # ALERT: Replace the namedtuple with a dict -- this way it can be added to the purl_dict as a nested dict rather than a list of 2 values. + closest_fixed_package_vulns_dict = [ + { + "vuln_id": fixed_pkg_vuln.vulnerability_id, + "vuln_get_absolute_url": fixed_pkg_vuln.get_absolute_url(), + } + for fixed_pkg_vuln in closest_fixed_package_vulns + ] + + # === + # closest_fixed_package_vulns_list = closest_fixed_package_vulns.objects.values_list() + + # closest_fixed_package_vulns_list = [] + # for closest_vuln in closest_fixed_package_vulns: + # closest_fixed_package_vulns_list.append(closest_vuln) + # print( + # "\t\nQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ vuln = {}\n".format( + # closest_vuln + # ) + # ) + # print("\t\ntype(closest_vuln) = {}".format(type(closest_vuln))) + + # # closest_fixed_package_vuln_count = len(closest_fixed_package.affected_by) + # print( + # "\t\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type(closest_fixed_package_vulns) = {}\n".format( + # type(closest_fixed_package_vulns) + # ) + # ) + + else: + closest_fixed_package = "There are no reported fixed packages." + # Is None the value we want? We do not want to display anything but the count = 0. + # closest_fixed_package_vulns = None + # closest_fixed_package_vuln_count = 0 + + # closest_fixed_package_vulns_list = [] + + print("\nzzz closest_fixed_package = {}".format(closest_fixed_package)) + print("zzz type(closest_fixed_package) = {}\n".format(type(closest_fixed_package))) + + # TODO: How do we add 'closest_fixed_by_purl', 'closest_fixed_by_vulns' and 'non_vulnerable_fix'? + + # # for vuln in purl_dict["vulnerabilities"]: + # # # vuln["closest_fixed_by_purl"] = "?????" + # # vuln["closest_fixed_by_purl"] = closest_fixed_package + # # vuln["closest_fixed_by_url"] = "?????" + # # vuln["closest_fixed_by_vulnerabilities"] = "?????" + # # vuln["non_vulnerable_fix"] = "?????" + # # vuln["non_vulnerable_fix_url"] = "?????" + + for dict_vuln in purl_dict["vulnerabilities"]: + print("\n===================================> vuln = {}\n".format(vuln)) + print("\n===================================> type(vuln) = {}\n".format(type(vuln))) + print("\n===================================> vuln.vcid = {}\n".format(vuln.vcid)) + print( + "\n===================================> dict_vuln['vulnerability'] = {}\n".format( + dict_vuln["vulnerability"] + ) + ) + # TODO: Up above we defined 'non_vuln_sibs' but we still need to remove those with less than version + # ALERT: remove less than versions from 'non_vuln_sibs' + # 2023-08-05 Saturday 20:30:47. Hopefully just wrote the code for that up above, with the new list 'later_non_vuln_sibs'. + + closest_non_vulnerable_fix = "" + # if len(non_vuln_sibs) > 0: + # closest_non_vulnerable_fix = self.sort_by_version(non_vuln_sibs)[0] + if len(later_non_vuln_sibs) > 0: + closest_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[0] + # else: + # # closest_non_vulnerable_fix = ( + # # "There are no reported non-vulnerable fixed packages." + # # ) + # closest_non_vulnerable_fix = None + + most_recent_non_vulnerable_fix = "" + if len(later_non_vuln_sibs) > 0: + most_recent_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[-1] + else: + # most_recent_non_vulnerable_fix = ( + # "There are no reported non-vulnerable fixed packages." + # ) + most_recent_non_vulnerable_fix = None + + # if dict_vuln["vulnerability"] == vuln.vulnerability_id: + if dict_vuln["vulnerability"] == str(vuln): + # if dict_vuln["vulnerability"] == vuln.vcid: + # dict_vuln["closest_fixed_by_purl"] = "?????" + dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) + dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() + # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vuln_count + # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns + # dict_vuln["closest_fixed_by_vulnerabilities"] = ["A", "B"] + + # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_list + # ALERT: Replace the above list created with a namedtuple with the following dictionary: + dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_dict + # ALERT: Moved these up 1 level in the dict. + # dict_vuln["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + # dict_vuln[ + # "closest_non_vulnerable_fix_url" + # ] = closest_non_vulnerable_fix.get_absolute_url() + # dict_vuln["most_recent_non_vulnerable_fix"] = str( + # most_recent_non_vulnerable_fix + # ) + # dict_vuln[ + # "most_recent_non_vulnerable_fix_url" + # ] = most_recent_non_vulnerable_fix.get_absolute_url() + + # QUESTION: Can we add the non-vuln data as higher-level key-value pairs rather than children of "vulnerabilities"? + + # purl_dict.update({"fruits": []}) + # purl_dict["fruits"].append({"fruit": "apple"}) + # purl_dict["fruits"].append({"fruit": "banana"}) + + # purl_dict.update( + # {"closest_non_vulnerable_fix": str(closest_non_vulnerable_fix)} + # ) + # purl_dict.update( + # { + # "closest_non_vulnerable_fix_url": closest_non_vulnerable_fix.get_absolute_url() + # } + # ) + # purl_dict.update( + # {"most_recent_non_vulnerable_fix": str(most_recent_non_vulnerable_fix)} + # ) + # purl_dict.update( + # { + # "most_recent_non_vulnerable_fix_url": most_recent_non_vulnerable_fix.get_absolute_url() + # } + # ) + + purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + purl_dict[ + "closest_non_vulnerable_fix_url" + ] = closest_non_vulnerable_fix.get_absolute_url() + purl_dict["most_recent_non_vulnerable_fix"] = str( + most_recent_non_vulnerable_fix + ) + purl_dict[ + "most_recent_non_vulnerable_fix_url" + ] = most_recent_non_vulnerable_fix.get_absolute_url() + + print("\npurl_dict = {}\n".format(purl_dict)) + + print(json.dumps(purl_dict, indent=4, sort_keys=False)) + + # # Print to text file + pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) + # logger = logging.getLogger(__name__) + # logger.setLevel(logging.INFO) + # # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt")) + # # will this overwrite prior writes? weird output + # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt", mode="w")) + # logger.info(pretty_purl_dict) + + with open("/home/jmh/pretty_purl_dict.txt", "w") as f: + f.write(pretty_purl_dict) + + alternate_dict_01 = { + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "vulnerabilities": [ + { + "vulnerability": "VCID-2nyb-8rwu-aaag", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + # "get_absolute_url": reverse("package_details", args=[self.purl]), + "closest_fixed_by_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2"], + ), + "closest_fixed_by_vulns": 2, + "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "non_vulnerable_fix_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + }, + { + "vulnerability": "VCID-gqhw-ngh8-aaap", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4", + # "get_absolute_url": reverse("package_details", args=[self.purl]), + "closest_fixed_by_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"], + ), + "closest_fixed_by_vulns": 1, + "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "non_vulnerable_fix_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + }, + { + "vulnerability": "VCID-t7e4-g3fr-aaan", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + # "get_absolute_url": reverse("package_details", args=[self.purl]), + "closest_fixed_by_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + "closest_fixed_by_vulns": 0, + "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "non_vulnerable_fix_url": reverse( + "package_details", + args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], + ), + }, + ], + } + + # return vcio_dict + + # return alternate_dict_01 + + return purl_dict class PackageRelatedVulnerability(models.Model): diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index e07ae199e..dab4283c7 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -41,7 +41,6 @@
    -
    During testing, red = all packages that fix the vulnerability (for pkg in vulnerability.fixed_by_packages) and green = the closest fixing package whose version is greater than the affected package's version (if pkg in get_fixing_packages).
    Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }})
    @@ -52,7 +51,7 @@ Vulnerability Summary Aliases - Fixing Packages + Fixed by packages @@ -76,22 +75,147 @@ {% endif %} {% endfor %} - + + + + + + + + {% if package.purl in fixed_package_details.purl %} + {% for key, value in fixed_package_details.items %} + {% if key == "vulnerabilities" %} + {% for abc in value %} + {% if abc.vulnerability == vulnerability.vulnerability_id %} +
      +
    • + PURL: {{ fixed_package_details.purl }} + + + +
      • +
    • + Vulnerability: {{ abc.vulnerability }} +
      • +
    • + Closest fixed-by PURL: + {{ abc.closest_fixed_by_purl }} +
      • +
    • + Closest fixed-by vulnerability count: {{ abc.closest_fixed_by_vulnerabilities|length }} + {% if abc.closest_fixed_by_vulnerabilities|length != 0 %} +
      +
      + +
      + +
      + {% endif %} +
      • +
    • + Closest non-vulnerable fix: + + + {{ fixed_package_details.closest_non_vulnerable_fix }} +
      • +
    • + Most recent non-vulnerable fix: + + + {{ fixed_package_details.most_recent_non_vulnerable_fix }} +
      • +
    + {% endif %} + {% endfor %} + + + {% endif %} + {% endfor %} + {% else %} + NO-- {{ package.purl }} + {% endif %} + + + + + +
    @@ -109,7 +233,7 @@
    - Fixing vulnerabilities ({{ fixing_vulnerabilities|length }}) + Fixed by vulnerabilities ({{ fixing_vulnerabilities|length }})
    diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 3daeca99a..94d1a6fb5 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -7,12 +7,14 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import urllib.parse from datetime import datetime from unittest import TestCase import pytest from django.db.utils import IntegrityError from freezegun import freeze_time +from univers import versions from vulnerabilities import models @@ -94,3 +96,66 @@ def test_cwe_not_present_in_weaknesses_db(self): assert w1.weakness is None assert w1.name is "" assert w1.description is "" + + +@pytest.mark.django_db +class TestPackageModel(TestCase): + def test_univers_version_comparisons(self): + assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") + + assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") + + # pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 is a real PURL in the DB + # But I get an error when I try to compare 2 PURLs with the same suffix -- + # univers.versions.InvalidVersion: '2.12.1-1%2Bdeb11u1' is not a valid + # Do we need to replace/delete the "%"? + # assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( + # "2.13.1-1%2Bdeb11u1" + # ) + # Test the error + with pytest.raises(versions.InvalidVersion): + assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( + "2.13.1-1%2Bdeb11u1" + ) + # Decode the version and test. + assert versions.DebianVersion( + urllib.parse.unquote("2.12.1-1%2Bdeb11u1") + ) < versions.DebianVersion(urllib.parse.unquote("2.13.1-1%2Bdeb11u1")) + + with pytest.raises(TypeError): + assert versions.PypiVersion("0.9") < versions.DebianVersion("0.10") + + # Using versions.Version does not correctly make this comparison! + assert not versions.Version("0.9") < versions.Version("0.10") + # Use SemverVersion instead as a default fallback version for comparisons. + assert versions.SemverVersion("0.9") < versions.SemverVersion("0.10") + + def test_assign_and_compare_univers_versions(self): + deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") + deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") + + immediate_fix01 = deb01.assign_and_compare_univers_versions(deb02) + print("\nimmediate_fix01 = {}\n".format(immediate_fix01)) + # assert deb01.assign_and_compare_univers_versions(deb02) is True + assert deb01.assign_and_compare_univers_versions(deb02) + + immediate_fix02 = deb02.assign_and_compare_univers_versions(deb01) + print("\nimmediate_fix02 = {}\n".format(immediate_fix02)) + # assert deb02.assign_and_compare_univers_versions(deb01) is False + assert not deb02.assign_and_compare_univers_versions(deb01) + + pypi01 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") + pypi02 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.10") + + immediate_fix03 = pypi01.assign_and_compare_univers_versions(pypi02) + print("\nimmediate_fix03 = {}\n".format(immediate_fix03)) + # assert pypi01.assign_and_compare_univers_versions(pypi02) is True + assert pypi01.assign_and_compare_univers_versions(pypi02) + + gem01 = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") + gem02 = models.Package.objects.create(type="gem", name="sidekiq", version="0.10") + + immediate_fix04 = gem01.assign_and_compare_univers_versions(gem02) + print("\nimmediate_fix04 = {}\n".format(immediate_fix04)) + # assert gem01.assign_and_compare_univers_versions(gem02) is True + assert gem01.assign_and_compare_univers_versions(gem02) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 1030fb410..5247c1c06 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -84,7 +84,10 @@ def get_context_data(self, **kwargs): context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) - context["get_fixing_packages"] = package.get_fixing_packages + # context["get_fixing_packages"] = package.get_fixing_packages + context["get_closest_fixed_package"] = package.get_closest_fixed_package + # context["test_property_01"] = package.test_property_01 + context["fixed_package_details"] = package.fixed_package_details return context diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 525127915..122da8025 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -39,7 +39,9 @@ CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS", default=[]) # SECURITY WARNING: do not run with debug turned on in production -DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) +# DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) +# DEBUG = "127.0.0.1" +DEBUG = True # SECURITY WARNING: do not run with debug turned on in production DEBUG_TOOLBAR = env.bool("VULNERABLECODE_DEBUG_TOOLBAR", default=False) diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index b80ba3257..205df5c63 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -327,6 +327,14 @@ a.small_page_button { box-shadow: 0px 8px 16px 0px #808080; } +.dropdown-vuln-list-width { + width: 400px; +} + +.dropdown-vuln-dict-width { + max-width: 600px; +} + .width-100-pct { width: 100%; } @@ -383,3 +391,94 @@ span.tag.custom { border: solid 1px #dbdbdb; background-color: #ffffff; } +/* test bulleted list */ + +ul.fixed_by_bullet { + list-style-type: disc; + /*margin-top: 2px; +margin-bottom: 10px;*/ + /*margin-left: -24px;*/ + /*margin-left: -30px;*/ + margin-top: 0.25em; + margin-left: 7px; + margin-bottom: 0.25em; + padding-left: 10px; +} + +ul.fixed_by_bullet ul { + list-style-type: disc; + /*margin-top: 10px;*/ + margin-top: 5px; + margin-top: 0px; + margin-bottom: 0px; + margin-left: 23px; + margin-left: 18px; + padding: 0; + border: none; +} + + + +ul.fixed_by_bullet li { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + /*margin-bottom: 10px;*/ + margin-bottom: 2px; +} + +ul.fixed_by_bullet li:last-child { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + /*margin-bottom: 10px;*/ + margin-bottom: 0px; +} + +ul.fixed_by_bullet li li { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + margin-top: 0px; + color: #000000; +} + +/* 10/10/15 add 3rd-level bullets */ +ul.fixed_by_bullet ul ul { + list-style-type: disc; + margin-top: 0px; + margin-bottom: 0px; + margin-left: 50px; + margin-left: 17px; + padding: 0; + border: none; +} + +ul.fixed_by_bullet li li li { + margin-left: 0px; + font-family: Arial; + font-family: BlinkMacSystemFont, -apple-system, "Segoe UI", "Roboto", "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", "Helvetica", "Arial", sans-serif; + font-size: 13px; + font-weight: normal; + margin-top: 0px; + color: #000000; +} + +/* CSS for dev fixed by headers */ +.dev_fixed_by_headers { + border: solid 1px #cccccc; + border-radius: 3px; + background-color: #f2f2f2; + color: #000000; + font-weight: bold; + font-size: 13px; + padding: 3px; + margin-bottom: 3px; + display: block; +} From 3b34e46cca5f70d15c858c55500c70d7918dbe34 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Wed, 9 Aug 2023 18:47:51 -0700 Subject: [PATCH 25/53] Improve matching and reporting code and UI #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 583 ++---------------- .../templates/package_details.html | 208 +++---- vulnerabilities/tests/test_models.py | 45 +- vulnerabilities/views.py | 3 - vulnerablecode/settings.py | 4 +- vulnerablecode/static/css/custom.css | 30 +- 6 files changed, 153 insertions(+), 720 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 55f2b00e3..494b045f9 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -660,7 +660,8 @@ def get_fixed_packages(self, package): def get_sibling_packages(self, package): """ - Return a queryset of all packages with the same type, namespace, name, subpath and qualifiers of the `package`, whether or not they fix any vulnerability + Return a queryset of all packages with the same type, namespace, name, subpath and + qualifiers as the target package, whether or not the 'sibling packages' fix any vulnerability. """ return Package.objects.filter( name=package.name, @@ -668,462 +669,126 @@ def get_sibling_packages(self, package): type=package.type, qualifiers=package.qualifiers, subpath=package.subpath, - # packagerelatedvulnerability__fix=True, ).distinct() - # def assign_and_compare_univers_versions(self, fixed_pkg): def assign_univers_version(self, fixed_pkg): """ - Identify which univers version applies to the two packages to be compared (self and a fixed package), - evaluate whether the fixed_pkg version is > than the target affected package, and - return True or False. + Identify which univers version applies to a package and return that version for use, e.g., + in sorting a group of sibling packages (same type etc.). """ - - # TODO: Instead of return True or False based on evaluating the incoming fixed_pkg type to a univers version and then checking whether the fixed version is greater than the affected (self) version, we'll just use the incoming type to assign and return a univers version -- as command_name -- to be used in get_closest_fixed_package() below to add all greater-than versions to the later_matching_fixed_packages list -- which in turn will be fed to self.sort_by_version(later_matching_fixed_packages) - - # Many more to be added. + # TODO: Many more to be added. match_type_to_univers_version = { "conan": versions.ConanVersion, "deb": versions.DebianVersion, + # The following throws an error: AttributeError: module 'univers.versions' has no attribute 'GemVersion' + # "gem": versions.GemVersion, "maven": versions.MavenVersion, + "nginx": versions.NginxVersion, + # "npm": "openssl": versions.OpensslVersion, "pypi": versions.PypiVersion, + # from https://github.com/nexB/univers/blob/205da7ecbf7b0f195662373ea710b2b84a877eb0/tests/test_version_comparison.py + # versions.SemverVersion, + # versions.GolangVersion, + # versions.PypiVersion, + # versions.GenericVersion, + # versions.ComposerVersion, + # versions.NginxVersion, + # versions.ArchLinuxVersion, + # versions.DebianVersion, + # versions.RpmVersion, + # versions.MavenVersion, + # versions.NugetVersion, + # versions.GentooVersion, + # versions.OpensslVersion, + # versions.LegacyOpensslVersion, + # versions.AlpineLinuxVersion, } command_name = "" - matched_type_to_version = match_type_to_univers_version.get(fixed_pkg.type) if matched_type_to_version: - print("\t--------------------------") - print("*** matched_type_to_version = {}".format(matched_type_to_version)) command_name = matched_type_to_version - else: - print("\t--------------------------") - print("*** matched_type_to_version = NO MATCH") - # Using "command_name = versions.Version", the test - # assert versions.Version("0.9") < versions.Version("0.10") - # fails! - # command_name = versions.Version - # Use this as a default fallback instead. command_name = versions.SemverVersion - # if command_name(fixed_pkg.version) > command_name(self.version): - # return True - # else: - # return False - - # Instead return command_name for recipient to use as needed for sorting or perhaps other uses return command_name def sort_by_version(self, later_matching_fixed_packages): # Incoming is a list of - - # ALERT: added this to address server error 500 but related error arose: line 908, in get_closest_fixed_package - # HOT: How is this related to the source of the server error (500)? - if len(later_matching_fixed_packages) == 0: - return - - # Replace find_closest_fixed_by_package()? - print("\nlater_matching_fixed_packages = {}".format(later_matching_fixed_packages)) - print("\nlater_matching_fixed_packages[0] = {}".format(later_matching_fixed_packages[0])) - print( - "\ntype(later_matching_fixed_packages[0]) = {}".format( - type(later_matching_fixed_packages[0]) - ) - ) - # NOTE: This gives us the PURL type but instead we want the PURL itself to pass to assign_univers_version(self, fixed_pkg) and get the command_name in return, which we'll then use in the sort process. - print( - "\nlater_matching_fixed_packages[0].type = {}".format( - later_matching_fixed_packages[0].type - ) - ) - print( - "\ntype(later_matching_fixed_packages[0].type) = {}".format( - type(later_matching_fixed_packages[0].type) - ) - ) - - # Incoming is a list -- later_matching_fixed_packages - # We'll use assign_univers_version() above to get the univers version as a command_name. - # But what do we pass to it? The [0] index of the incoming list, i.e., later_matching_fixed_packages[0]? command_name = self.assign_univers_version(later_matching_fixed_packages[0]) - - print("\n>>> command_name = {}\n".format(command_name)) - - # TODO: Maybe we don't need to convert to a PURL, a list of dictionaries etc.?? - print( - "\n+++++++ later_matching_fixed_packages[0].version = {}".format( - later_matching_fixed_packages[0].version - ) - ) - - # sort test_sort_by_version = [] test_sort_by_version = sorted( - # later_matching_fixed_packages, key=lambda x: versions.DebianVersion(x["version"]) later_matching_fixed_packages, - # key=lambda x: versions.MavenVersion(x.version), key=lambda x: command_name(x.version), ) - print("\ntest_sort_by_version = {}\n".format(test_sort_by_version)) - return test_sort_by_version - # convert_to_dict_list = [] - - # sorted_later_matching_fixed_packages = [] - - # # TODO: First, convert to a list of dictionaries. - # for pkg in later_matching_fixed_packages: - # # pkg is a - # print("pkg = {}".format(pkg)) - # print("type(pkg) = {}".format(type(pkg))) - - # # pkg_str is a string - # pkg_str = pkg.package_url - # print("pkg_str = {}".format(pkg_str)) - # print("type(pkg_str) = {}".format(type(pkg_str))) - - # # purl is a - # purl = PackageURL.from_string(pkg_str) - # print("purl = {}".format(purl)) - # print("type(purl) = {}".format(type(purl))) - - # purl_dict = purl.to_dict() - # print("purl_dict = {}".format(purl_dict)) - # print("type(purl_dict) = {}".format(type(purl_dict))) - - # convert_to_dict_list.append(purl.to_dict()) - # print("HELLO\n") - - # print("\nconvert_to_dict_list = {}\n".format(convert_to_dict_list)) - - # return convert_to_dict_list - # ========================================================== - # sorted_later_matching_fixed_packages = sorted( - # later_matching_fixed_packages, key=lambda x: versions.MavenVersion(x["version"]) - # ) - # print( - # "\nsorted_later_matching_fixed_packages = {}\n".format( - # sorted_later_matching_fixed_packages - # ) - # ) - # print("\n".join(map(str, sorted_later_matching_fixed_packages))) - - # return what? - - # ========================================================== - # ========================================================== - - # def find_closest_fixed_by_package(self, later_matching_fixed_packages): - # # Maybe use sort_by_version() above instead? - # # take the incoming list later_matching_fixed_packages, convert to list of dictionaries, sort by version using univers.version.[version class], choose the top i.e., index [0] and convert back to PURL and return that PURL. - # print("\nlater_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) - - # closest_fixed_by_package = "TBD" - - # return closest_fixed_by_package - - @property - # def get_fixing_packages(self): - def get_closest_fixed_package(self): - """ - This function identifies the closest fixed package version that is greater than the affected package version and - is the same type, namespace, name, qualifiers and subpath as the affected package. - """ - - print("\nself = {}\n".format(self)) - - # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. - # fixed_packages = self.get_fixed_packages(package=self) - # This is clearer. - matching_fixed_packages = self.get_fixed_packages(package=self) - - # This returns a list of the vulnerabilities that affect this package (i.e., self). - qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - - # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). - # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). - qs = qs.prefetch_related( - Prefetch( - "packages", - # queryset=fixed_packages, - queryset=matching_fixed_packages, - # to_attr="filtered_fixed_packages", - to_attr="matching_fixed_packages", - ) - ) - - # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). - print("qs = {}\n".format(qs)) - - # ************************************************************************ - - later_matching_fixed_packages = [] - - vuln_count = 0 - for vuln in qs: - print("vuln = {}\n".format(vuln)) - # print( - # "\tqs[vuln_count].filtered_fixed_packages = {}".format( - # qs[vuln_count].filtered_fixed_packages - # ) - # ) - print( - "\tqs[vuln_count].matching_fixed_packages = {}".format( - qs[vuln_count].matching_fixed_packages - ) - ) - print("") - - # Check the Prefetch qs. - # TODO: Do we want to check whether the fixed version has any vulnerabilities of its own? - # for fixed_pkg in qs[vuln_count].filtered_fixed_packages: - for fixed_pkg in qs[vuln_count].matching_fixed_packages: - print("\tfixed_pkg = {}".format(fixed_pkg)) - print("\tfixed_pkg.type = {}".format(fixed_pkg.type)) - print("\tfixed_pkg.version = {}".format(fixed_pkg.version)) - print("\t--------------------------") - print("\tself.type = {}".format(self.type)) - print("\tself.version = {}".format(self.version)) - - # Assign univers version and compare: False = fixed_pkg.version < self.version (affected version). - - # 2023-08-02 Wednesday 16:01:35. atm immediate_fix is True or False. If instead assign_and_compare_univers_versions() returns the univers version, we could get that here and then test with this or similar right here -- enabling use of the univers version function in other places as well, like a sort_by_version function! - # if command_name(fixed_pkg.version) > command_name(self.version): - # return True - # else: - # return False - # ===================================================== - # Replace this with chunk below - # immediate_fix = self.assign_and_compare_univers_versions(fixed_pkg) - # print("\t--------------------------") - # print("\timmediate_fix = {}\n".format(immediate_fix)) - - # if fixed_pkg in fixed_packages and immediate_fix: - # later_matching_fixed_packages.append(fixed_pkg) - # ===================================================== - # command_name = self.assign_and_compare_univers_versions(fixed_pkg) - # renamed - # TODO: Move this up before the for loop -- both for loops if possible -- to reduce calls! - command_name = self.assign_univers_version(fixed_pkg) - print("\nJust requested command_name >>> {}\n".format(command_name)) - # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( - # self.version - # ): - if fixed_pkg in matching_fixed_packages and command_name( - fixed_pkg.version - ) > command_name(self.version): - later_matching_fixed_packages.append(fixed_pkg) - - vuln_count += 1 - - # find_closest_fixed_by_package -- from the list later_matching_fixed_packages - # closest_fixed_by_package = self.find_closest_fixed_by_package(later_matching_fixed_packages) - - # TODO: or instead use this. This will be a list sorted by univers version class, and here all we need is to grab the [0] index from that list for the closest fixed by package! So we'd return a single closest_fixed_package. - # ALERT: The sort query needs to be done separately for each vulnerability because the list of fixed by packages is likely to be different. As is, we return a single sorted list of all fixed by packages for the affected package and then pass just the [0] package -- not what we want to do! - sort_fixed_by_packages_by_version = self.sort_by_version(later_matching_fixed_packages) - print( - "\nsort_fixed_by_packages_by_version = {}\n".format(sort_fixed_by_packages_by_version) - ) - # ALERT: 2023-08-05 Saturday 23:22:08. Address server error 500? - # ALERT: 2023-08-05 Saturday 23:24:50. This actusally fixed the server error (500) and I can now even see the Packafe details page for pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1 !!! - # HOT: I need to trace back the root cause of the server error (500). I suspect it's something like a record with no fixed by packages or something else that is an empty list but which i try to measure, e.g., for a print statement, or possibly for a real if condition.. - if sort_fixed_by_packages_by_version is None: - return - - closest_fixed_package = sort_fixed_by_packages_by_version[0] - - # print("\n!!! later_matching_fixed_packages = {}\n".format(later_matching_fixed_packages)) - # print( - # "\n!!! sort_fixed_by_packages_by_version = {}\n".format( - # sort_fixed_by_packages_by_version - # ) - # ) - # print("\n!!! closest_fixed_package = {}\n".format(closest_fixed_package)) - - # rebuilt_purl_from_dict = PackageURL( - # closest_fixed_package["type"], - # closest_fixed_package["namespace"], - # closest_fixed_package["name"], - # closest_fixed_package["version"], - # closest_fixed_package["qualifiers"], - # closest_fixed_package["subpath"], - # ) - # print("\n!!! rebuilt_purl_from_dict = {}\n".format(rebuilt_purl_from_dict)) - - # return later_matching_fixed_packages - return sort_fixed_by_packages_by_version - # return [closest_fixed_package] - - # return [rebuilt_purl_from_dict] - @property def fixed_package_details(self): """ - This is a test that might develop into a model-based equivalent of the loops etc. I was doing/trying to do in the Jinja2 template. I'm going to add this as a context so we can see it in the template. + Find and report all fixed by packages (and certain related versions) for each vulnerability in + each affected package. Report packages with no vulnerabilities as well. """ - # return "Hello" - - # vcio_dict = { - # [ - # {"VCID-2nyb-8rwu-aaag": "PURL01"}, - # {"VCID-gqhw-ngh8-aaap": "PURL02"}, - # {"some-other-id": "PURL03"}, - # ] - # } - - print("\n==> This is from the test_property_01() property.\n") - - print("\nself = {}\n".format(self)) - - # This returns all fixed packages that match the target package (type etc.), regardless of fixed vuln. - # fixed_packages = self.get_fixed_packages(package=self) - # This is clearer. + # Get all fixed packages that match the target package (type etc.), regardless of fixed vuln. matching_fixed_packages = self.get_fixed_packages(package=self) - # This returns a list of the vulnerabilities that affect this package (i.e., self). + # Get a list of the vulnerabilities that affect this package (i.e., self). qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - # TODO: Can we get all sibling packages so that we can then determine which have 0 vulnerabilities and of these the closest and maybe the most recent as well? - all_sibling_packages = self.get_sibling_packages(package=self) - print("\nall_sibling_packages = {}\n".format(all_sibling_packages)) - print("\nlen(all_sibling_packages) = {}\n".format(len(all_sibling_packages))) non_vuln_sibs = [] for sib in all_sibling_packages: if sib.is_vulnerable is False: non_vuln_sibs.append(sib) - print("\nnon_vuln_sibs = {}\n".format(non_vuln_sibs)) - print("\nlen(non_vuln_sibs) = {}\n".format(len(non_vuln_sibs))) # Add just the greater-than versions to a new list command_name = self.assign_univers_version(self) - print( - "\nOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO command_name = {}\n".format( - command_name - ) - ) + later_non_vuln_sibs = [] for non_vuln_sib in non_vuln_sibs: if command_name(non_vuln_sib.version) > command_name(self.version): later_non_vuln_sibs.append(non_vuln_sib) - print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) - print("\nlen(later_non_vuln_sibs) = {}\n".format(len(later_non_vuln_sibs))) - - # This takes the list of vulns affecting the current package, retrieves a list of the fixed packages for each vuln, and assigns the result to a custom attribute, `filtered_fixed_packages` (renamed 'matching_fixed_packages'). - # We use this in a for loop below like this -- qs[vuln_count].filtered_fixed_packages (renamed 'matching_fixed_packages') -- where `vuln_count` is used to iterate through the list of vulns that affect the current package (i.e., self). + # Take the list of vulns affecting the current package, retrieve a list of the fixed packages for each vuln, and assign the result to a custom attribute, 'matching_fixed_packages'. Ex: qs[0].matching_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). qs = qs.prefetch_related( Prefetch( "packages", - # queryset=fixed_packages, queryset=matching_fixed_packages, - # to_attr="filtered_fixed_packages", to_attr="matching_fixed_packages", ) ) - # Ex: qs[0].filtered_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). - print("\nzzz qs = {}\n".format(qs)) - purl_dict = {} - purl_dict["purl"] = self.purl - purl_dict.update({"vulnerabilities": []}) - # purl_dict["vulnerabilities"].append({"fruit": "orange"}) + # HOT: In constructing the purl_dict, I need to include packages that have no vulnerabilities -- we still want various dict fields to be populated to reflect the relevant structure and content all such packages. atm only a few fields are included in the dict for such a package. "closest_non_vulnerable_fix", "closest_non_vulnerable_fix_url", "most_recent_non_vulnerable_fix" and "most_recent_non_vulnerable_fix_url" are currently omitted. for vuln in qs: - print("\nzzz vuln = {}\n".format(vuln)) - print("\nzzz type(vuln) = {}\n".format(type(vuln))) - later_matching_fixed_packages = [] - # purl_dict[vuln.vulnerability_id] = "aaa" - # purl_dict.update({"vulnerability": vuln.vulnerability_id}) - purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) - - # TODO:2023-08-05 Saturday 13:12:28. This returns a list of matching fixed packages for this specific vuln! vuln_matching_fixed_packages = vuln.matching_fixed_packages - print("\nzzz self.purl = {}\n".format(self.purl)) - print("\nzzz vuln = {}\n".format(vuln)) - print("\nzzz vuln_matching_fixed_packages = {}\n".format(vuln_matching_fixed_packages)) - - # TODO: So we need to sort this list by version using the correct univers version and then return the [0] index in that sorted list - # QUESTION: Do we still need to remove lesser-than fixed packages or did we already do that? - - # ============================================================= - # command_name = self.assign_univers_version(fixed_pkg) command_name = self.assign_univers_version(self) - print("\nzzz command_name = {}\n".format(command_name)) - - # ALERT: What if there are no fixed by packages? The following thows an error because the list 'vuln_matching_fixed_packages' is empty! - # [I fixed this, right? ;-] - closest_fixed_package = "" if len(vuln_matching_fixed_packages) > 0: - for fixed_pkg in vuln_matching_fixed_packages: if fixed_pkg in matching_fixed_packages and command_name( fixed_pkg.version ) > command_name(self.version): later_matching_fixed_packages.append(fixed_pkg) - # print("\nJust requested command_name >>> {}\n".format(command_name)) - # # if fixed_pkg in fixed_packages and command_name(fixed_pkg.version) > command_name( - # # self.version - # # ): - # if fixed_pkg in matching_fixed_packages and command_name( - # fixed_pkg.version - # ) > command_name(self.version): - # later_matching_fixed_packages.append(fixed_pkg) - # ============================================================= - # later_matching_fixed_packages = vuln.matching_fixed_packages - - print( - "\nzzz later_matching_fixed_packages = {}\n".format( - later_matching_fixed_packages - ) - ) - sort_fixed_by_packages_by_version = self.sort_by_version( later_matching_fixed_packages ) - print( - "\nzzz sort_fixed_by_packages_by_version = {}\n".format( - sort_fixed_by_packages_by_version - ) - ) + closest_fixed_package = sort_fixed_by_packages_by_version[0] - # closest_fixed_package = sort_fixed_by_packages_by_version[0].purl - # 2023-08-06 Sunday 11:15:03. This returns a queryset of vulns affecting this package. - # HOT: How do we get the closest fixed by package vuln count and list of vulns? I keep getting errors. ;-) closest_fixed_package_vulns = closest_fixed_package.affected_by - # closest_fixed_package_vulns_list = list(closest_fixed_package_vulns) - # ALERT: 2023-08-06 Sunday 14:25:49. This did the trick! - - # closest_fixed_package_vulns_list = [ - # i.vulnerability_id for i in closest_fixed_package_vulns - # ] - - # 2023-08-06 Sunday 16:53:27. Try a named tuple to pass the vuln's vulnerability+id and get_absolute_url. - # FixedPackageVuln = namedtuple("FixedPackageVuln", "vuln_id, vuln_get_absolute_url") - # closest_fixed_package_vulns_list = [ - # FixedPackageVuln( - # vuln_id=fixed_pkg_vuln.vulnerability_id, - # vuln_get_absolute_url=fixed_pkg_vuln.get_absolute_url(), - # ) - # for fixed_pkg_vuln in closest_fixed_package_vulns - # ] - # ALERT: Replace the namedtuple with a dict -- this way it can be added to the purl_dict as a nested dict rather than a list of 2 values. + closest_fixed_package_vulns_dict = [ { "vuln_id": fixed_pkg_vuln.vulnerability_id, @@ -1132,127 +797,25 @@ def fixed_package_details(self): for fixed_pkg_vuln in closest_fixed_package_vulns ] - # === - # closest_fixed_package_vulns_list = closest_fixed_package_vulns.objects.values_list() - - # closest_fixed_package_vulns_list = [] - # for closest_vuln in closest_fixed_package_vulns: - # closest_fixed_package_vulns_list.append(closest_vuln) - # print( - # "\t\nQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQQ vuln = {}\n".format( - # closest_vuln - # ) - # ) - # print("\t\ntype(closest_vuln) = {}".format(type(closest_vuln))) - - # # closest_fixed_package_vuln_count = len(closest_fixed_package.affected_by) - # print( - # "\t\n^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type(closest_fixed_package_vulns) = {}\n".format( - # type(closest_fixed_package_vulns) - # ) - # ) - else: closest_fixed_package = "There are no reported fixed packages." - # Is None the value we want? We do not want to display anything but the count = 0. - # closest_fixed_package_vulns = None - # closest_fixed_package_vuln_count = 0 - - # closest_fixed_package_vulns_list = [] - - print("\nzzz closest_fixed_package = {}".format(closest_fixed_package)) - print("zzz type(closest_fixed_package) = {}\n".format(type(closest_fixed_package))) - - # TODO: How do we add 'closest_fixed_by_purl', 'closest_fixed_by_vulns' and 'non_vulnerable_fix'? - - # # for vuln in purl_dict["vulnerabilities"]: - # # # vuln["closest_fixed_by_purl"] = "?????" - # # vuln["closest_fixed_by_purl"] = closest_fixed_package - # # vuln["closest_fixed_by_url"] = "?????" - # # vuln["closest_fixed_by_vulnerabilities"] = "?????" - # # vuln["non_vulnerable_fix"] = "?????" - # # vuln["non_vulnerable_fix_url"] = "?????" for dict_vuln in purl_dict["vulnerabilities"]: - print("\n===================================> vuln = {}\n".format(vuln)) - print("\n===================================> type(vuln) = {}\n".format(type(vuln))) - print("\n===================================> vuln.vcid = {}\n".format(vuln.vcid)) - print( - "\n===================================> dict_vuln['vulnerability'] = {}\n".format( - dict_vuln["vulnerability"] - ) - ) - # TODO: Up above we defined 'non_vuln_sibs' but we still need to remove those with less than version - # ALERT: remove less than versions from 'non_vuln_sibs' - # 2023-08-05 Saturday 20:30:47. Hopefully just wrote the code for that up above, with the new list 'later_non_vuln_sibs'. - closest_non_vulnerable_fix = "" - # if len(non_vuln_sibs) > 0: - # closest_non_vulnerable_fix = self.sort_by_version(non_vuln_sibs)[0] + if len(later_non_vuln_sibs) > 0: closest_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[0] - # else: - # # closest_non_vulnerable_fix = ( - # # "There are no reported non-vulnerable fixed packages." - # # ) - # closest_non_vulnerable_fix = None most_recent_non_vulnerable_fix = "" if len(later_non_vuln_sibs) > 0: most_recent_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[-1] else: - # most_recent_non_vulnerable_fix = ( - # "There are no reported non-vulnerable fixed packages." - # ) most_recent_non_vulnerable_fix = None - # if dict_vuln["vulnerability"] == vuln.vulnerability_id: if dict_vuln["vulnerability"] == str(vuln): - # if dict_vuln["vulnerability"] == vuln.vcid: - # dict_vuln["closest_fixed_by_purl"] = "?????" dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() - # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vuln_count - # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns - # dict_vuln["closest_fixed_by_vulnerabilities"] = ["A", "B"] - - # dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_list - # ALERT: Replace the above list created with a namedtuple with the following dictionary: dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_dict - # ALERT: Moved these up 1 level in the dict. - # dict_vuln["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) - # dict_vuln[ - # "closest_non_vulnerable_fix_url" - # ] = closest_non_vulnerable_fix.get_absolute_url() - # dict_vuln["most_recent_non_vulnerable_fix"] = str( - # most_recent_non_vulnerable_fix - # ) - # dict_vuln[ - # "most_recent_non_vulnerable_fix_url" - # ] = most_recent_non_vulnerable_fix.get_absolute_url() - - # QUESTION: Can we add the non-vuln data as higher-level key-value pairs rather than children of "vulnerabilities"? - - # purl_dict.update({"fruits": []}) - # purl_dict["fruits"].append({"fruit": "apple"}) - # purl_dict["fruits"].append({"fruit": "banana"}) - - # purl_dict.update( - # {"closest_non_vulnerable_fix": str(closest_non_vulnerable_fix)} - # ) - # purl_dict.update( - # { - # "closest_non_vulnerable_fix_url": closest_non_vulnerable_fix.get_absolute_url() - # } - # ) - # purl_dict.update( - # {"most_recent_non_vulnerable_fix": str(most_recent_non_vulnerable_fix)} - # ) - # purl_dict.update( - # { - # "most_recent_non_vulnerable_fix_url": most_recent_non_vulnerable_fix.get_absolute_url() - # } - # ) purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) purl_dict[ @@ -1265,76 +828,14 @@ def fixed_package_details(self): "most_recent_non_vulnerable_fix_url" ] = most_recent_non_vulnerable_fix.get_absolute_url() - print("\npurl_dict = {}\n".format(purl_dict)) - - print(json.dumps(purl_dict, indent=4, sort_keys=False)) - - # # Print to text file - pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) - # logger = logging.getLogger(__name__) - # logger.setLevel(logging.INFO) - # # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt")) - # # will this overwrite prior writes? weird output - # logger.addHandler(logging.FileHandler("2023-08-07-pretty_purl_dict.txt", mode="w")) - # logger.info(pretty_purl_dict) - - with open("/home/jmh/pretty_purl_dict.txt", "w") as f: - f.write(pretty_purl_dict) - - alternate_dict_01 = { - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", - "vulnerabilities": [ - { - "vulnerability": "VCID-2nyb-8rwu-aaag", - "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - # "get_absolute_url": reverse("package_details", args=[self.purl]), - "closest_fixed_by_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2"], - ), - "closest_fixed_by_vulns": 2, - "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "non_vulnerable_fix_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], - ), - }, - { - "vulnerability": "VCID-gqhw-ngh8-aaap", - "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4", - # "get_absolute_url": reverse("package_details", args=[self.purl]), - "closest_fixed_by_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"], - ), - "closest_fixed_by_vulns": 1, - "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "non_vulnerable_fix_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], - ), - }, - { - "vulnerability": "VCID-t7e4-g3fr-aaan", - "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - # "get_absolute_url": reverse("package_details", args=[self.purl]), - "closest_fixed_by_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], - ), - "closest_fixed_by_vulns": 0, - "non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "non_vulnerable_fix_url": reverse( - "package_details", - args=["pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1"], - ), - }, - ], - } - - # return vcio_dict + # Temporary print output during dev/testing: + # print("\npurl_dict = {}\n".format(purl_dict)) + # print(json.dumps(purl_dict, indent=4, sort_keys=False)) - # return alternate_dict_01 + # TODO: Consider whether we want to provide the user with an option to output the dictionary to a file. + # pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) + # with open("pretty_purl_dict.txt", "w") as f: + # f.write(pretty_purl_dict) return purl_dict diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index dab4283c7..772a2d3f5 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -40,18 +40,47 @@
    + {% if affected_by_vulnerabilities|length != 0 %} + +
    + + + + + + + + + + + +
    + Closest non-vulnerable purl + + {{ fixed_package_details.closest_non_vulnerable_fix }} +
    + Latest non-vulnerable purl + + {{ fixed_package_details.most_recent_non_vulnerable_fix }} +
    +
    + + {% endif %} +
    Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }})
    + + - + @@ -75,159 +104,63 @@ {% endif %} {% endfor %} - {% empty %} - + {% endfor %} -
    Vulnerability Summary AliasesFixed by packagesClosest fixed by packages
    - - - - - - - - {% if package.purl in fixed_package_details.purl %} - {% for key, value in fixed_package_details.items %} - {% if key == "vulnerabilities" %} + + {% if package.purl in fixed_package_details.purl %} + {% for key, value in fixed_package_details.items %} + {% if key == "vulnerabilities" %} {% for abc in value %} {% if abc.vulnerability == vulnerability.vulnerability_id %} -
      -
    • - PURL: {{ fixed_package_details.purl }} - - - -
      • -
    • - Vulnerability: {{ abc.vulnerability }} -
      • -
    • - Closest fixed-by PURL: - {{ abc.closest_fixed_by_purl }} -
      • -
    • - Closest fixed-by vulnerability count: {{ abc.closest_fixed_by_vulnerabilities|length }} - {% if abc.closest_fixed_by_vulnerabilities|length != 0 %} -
      -
      - -
      - -
      - {% endif %} -
      • -
    • - Closest non-vulnerable fix: - - - {{ fixed_package_details.closest_non_vulnerable_fix }} -
      • -
    • - Most recent non-vulnerable fix: - - - {{ fixed_package_details.most_recent_non_vulnerable_fix }} -
      • -
    {% endif %} {% endfor %} - - - {% endif %} - {% endfor %} - {% else %} - NO-- {{ package.purl }} {% endif %} + {% endfor %} - - - - - - + {% endif %}
    This package is not known to be affected by vulnerabilities.
    @@ -244,7 +177,6 @@ Aliases - {% for vulnerability in fixing_vulnerabilities %} diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 94d1a6fb5..24bf0e166 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -102,60 +102,43 @@ def test_cwe_not_present_in_weaknesses_db(self): class TestPackageModel(TestCase): def test_univers_version_comparisons(self): assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") - assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") # pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 is a real PURL in the DB - # But I get an error when I try to compare 2 PURLs with the same suffix -- - # univers.versions.InvalidVersion: '2.12.1-1%2Bdeb11u1' is not a valid - # Do we need to replace/delete the "%"? - # assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( - # "2.13.1-1%2Bdeb11u1" - # ) - # Test the error + # But we need to replace/delete the "%". Test the error: with pytest.raises(versions.InvalidVersion): assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( "2.13.1-1%2Bdeb11u1" ) - # Decode the version and test. + # Decode the version and test: assert versions.DebianVersion( urllib.parse.unquote("2.12.1-1%2Bdeb11u1") ) < versions.DebianVersion(urllib.parse.unquote("2.13.1-1%2Bdeb11u1")) + # Expect an error when comparing different types. with pytest.raises(TypeError): assert versions.PypiVersion("0.9") < versions.DebianVersion("0.10") - # Using versions.Version does not correctly make this comparison! + # This demonstrates that versions.Version does not correctly compare 0.9 vs. 0.10. assert not versions.Version("0.9") < versions.Version("0.10") # Use SemverVersion instead as a default fallback version for comparisons. assert versions.SemverVersion("0.9") < versions.SemverVersion("0.10") - def test_assign_and_compare_univers_versions(self): deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") + assert versions.DebianVersion(deb01.version) < versions.DebianVersion(deb02.version) - immediate_fix01 = deb01.assign_and_compare_univers_versions(deb02) - print("\nimmediate_fix01 = {}\n".format(immediate_fix01)) - # assert deb01.assign_and_compare_univers_versions(deb02) is True - assert deb01.assign_and_compare_univers_versions(deb02) + def test_assign_univers_version(self): + requesting_package = models.Package.objects.create(type="deb", name="git", version="2.30.1") - immediate_fix02 = deb02.assign_and_compare_univers_versions(deb01) - print("\nimmediate_fix02 = {}\n".format(immediate_fix02)) - # assert deb02.assign_and_compare_univers_versions(deb01) is False - assert not deb02.assign_and_compare_univers_versions(deb01) + deb01 = models.Package.objects.create(type="deb", name="git", version="2.31.1") + command_name_deb01 = requesting_package.assign_univers_version(deb01) + assert command_name_deb01 == versions.DebianVersion pypi01 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") - pypi02 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.10") - - immediate_fix03 = pypi01.assign_and_compare_univers_versions(pypi02) - print("\nimmediate_fix03 = {}\n".format(immediate_fix03)) - # assert pypi01.assign_and_compare_univers_versions(pypi02) is True - assert pypi01.assign_and_compare_univers_versions(pypi02) + command_name_pypi01 = requesting_package.assign_univers_version(pypi01) + assert command_name_pypi01 == versions.PypiVersion gem01 = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") - gem02 = models.Package.objects.create(type="gem", name="sidekiq", version="0.10") - - immediate_fix04 = gem01.assign_and_compare_univers_versions(gem02) - print("\nimmediate_fix04 = {}\n".format(immediate_fix04)) - # assert gem01.assign_and_compare_univers_versions(gem02) is True - assert gem01.assign_and_compare_univers_versions(gem02) + command_name_gem01 = requesting_package.assign_univers_version(gem01) + assert command_name_gem01 == versions.SemverVersion diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 5247c1c06..7618369e2 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -84,9 +84,6 @@ def get_context_data(self, **kwargs): context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") context["package_search_form"] = PackageSearchForm(self.request.GET) - # context["get_fixing_packages"] = package.get_fixing_packages - context["get_closest_fixed_package"] = package.get_closest_fixed_package - # context["test_property_01"] = package.test_property_01 context["fixed_package_details"] = package.fixed_package_details return context diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 122da8025..525127915 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -39,9 +39,7 @@ CSRF_TRUSTED_ORIGINS = env.list("CSRF_TRUSTED_ORIGINS", default=[]) # SECURITY WARNING: do not run with debug turned on in production -# DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) -# DEBUG = "127.0.0.1" -DEBUG = True +DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) # SECURITY WARNING: do not run with debug turned on in production DEBUG_TOOLBAR = env.bool("VULNERABLECODE_DEBUG_TOOLBAR", default=False) diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index 205df5c63..bf094f70c 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -196,16 +196,14 @@ code { } .two-col-left { - width: 160px; + width: 250px; text-align: right !important; font-weight: bold; - padding-right: 20px !important; + padding-right: 15px !important; line-height: 20px; - border: solid 1px #e8e8e8 !important; } .two-col-right { - border: solid 1px #e8e8e8 !important; line-height: 20px; } @@ -482,3 +480,27 @@ ul.fixed_by_bullet li li li { margin-bottom: 3px; display: block; } + +.non-floating-purl { + position: relative; + width: 100%; + z-index: 100; + margin-bottom: 0px; +} + +.non-floating-purl .table td, +.non-floating-purl .table tbody tr:last-child td, +.non-floating-purl .table th { + border: solid 1px #dbdbdb; + background-color: #ffffff; +} + +.non-vuln { + margin-top: -25px; +} + +.non-vuln .table td, +.non-vuln .table tbody tr:last-child td, +.non-vuln .table th { + border: solid 1px #dbdbdb; +} From eb69a014f299c557b626e3b971d802437e6a780f Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 14 Aug 2023 15:04:38 -0700 Subject: [PATCH 26/53] Add univers version, revise sort and related code, update and add new tests #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 117 +++++------ vulnerabilities/tests/test_models.py | 292 +++++++++++++++++++++++++-- 2 files changed, 326 insertions(+), 83 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 494b045f9..d0824610b 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -34,6 +34,7 @@ from packageurl.contrib.django.models import without_empty_values from rest_framework.authtoken.models import Token from univers import versions +from univers.version_range import RANGE_CLASS_BY_SCHEMES from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import build_vcid @@ -671,60 +672,16 @@ def get_sibling_packages(self, package): subpath=package.subpath, ).distinct() - def assign_univers_version(self, fixed_pkg): - """ - Identify which univers version applies to a package and return that version for use, e.g., - in sorting a group of sibling packages (same type etc.). - """ - # TODO: Many more to be added. - match_type_to_univers_version = { - "conan": versions.ConanVersion, - "deb": versions.DebianVersion, - # The following throws an error: AttributeError: module 'univers.versions' has no attribute 'GemVersion' - # "gem": versions.GemVersion, - "maven": versions.MavenVersion, - "nginx": versions.NginxVersion, - # "npm": - "openssl": versions.OpensslVersion, - "pypi": versions.PypiVersion, - # from https://github.com/nexB/univers/blob/205da7ecbf7b0f195662373ea710b2b84a877eb0/tests/test_version_comparison.py - # versions.SemverVersion, - # versions.GolangVersion, - # versions.PypiVersion, - # versions.GenericVersion, - # versions.ComposerVersion, - # versions.NginxVersion, - # versions.ArchLinuxVersion, - # versions.DebianVersion, - # versions.RpmVersion, - # versions.MavenVersion, - # versions.NugetVersion, - # versions.GentooVersion, - # versions.OpensslVersion, - # versions.LegacyOpensslVersion, - # versions.AlpineLinuxVersion, - } - - command_name = "" - matched_type_to_version = match_type_to_univers_version.get(fixed_pkg.type) - if matched_type_to_version: - command_name = matched_type_to_version - else: - command_name = versions.SemverVersion - - return command_name - - def sort_by_version(self, later_matching_fixed_packages): + def sort_by_version(self, packages_to_sort): # Incoming is a list of - # We'll use assign_univers_version() above to get the univers version as a command_name. - command_name = self.assign_univers_version(later_matching_fixed_packages[0]) - test_sort_by_version = [] - test_sort_by_version = sorted( - later_matching_fixed_packages, - key=lambda x: command_name(x.version), + univers_version = RANGE_CLASS_BY_SCHEMES[packages_to_sort[0].type].version_class + sorted_by_version = [] + sorted_by_version = sorted( + packages_to_sort, + key=lambda x: univers_version(x.version), ) - return test_sort_by_version + return sorted_by_version @property def fixed_package_details(self): @@ -746,11 +703,10 @@ def fixed_package_details(self): non_vuln_sibs.append(sib) # Add just the greater-than versions to a new list - command_name = self.assign_univers_version(self) - + univers_version = RANGE_CLASS_BY_SCHEMES[self.type].version_class later_non_vuln_sibs = [] for non_vuln_sib in non_vuln_sibs: - if command_name(non_vuln_sib.version) > command_name(self.version): + if univers_version(non_vuln_sib.version) > univers_version(self.version): later_non_vuln_sibs.append(non_vuln_sib) # Take the list of vulns affecting the current package, retrieve a list of the fixed packages for each vuln, and assign the result to a custom attribute, 'matching_fixed_packages'. Ex: qs[0].matching_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). @@ -766,20 +722,24 @@ def fixed_package_details(self): purl_dict["purl"] = self.purl purl_dict.update({"vulnerabilities": []}) - # HOT: In constructing the purl_dict, I need to include packages that have no vulnerabilities -- we still want various dict fields to be populated to reflect the relevant structure and content all such packages. atm only a few fields are included in the dict for such a package. "closest_non_vulnerable_fix", "closest_non_vulnerable_fix_url", "most_recent_non_vulnerable_fix" and "most_recent_non_vulnerable_fix_url" are currently omitted. + purl_dict["closest_non_vulnerable_fix"] = "" + purl_dict["closest_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix"] = "" + purl_dict["most_recent_non_vulnerable_fix_url"] = "" for vuln in qs: later_matching_fixed_packages = [] purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) vuln_matching_fixed_packages = vuln.matching_fixed_packages - command_name = self.assign_univers_version(self) closest_fixed_package = "" + # Need to define here to avoid "" vs. [] for some closest_fixed_by_vulnerabilities values? + closest_fixed_package_vulns_dict = [] if len(vuln_matching_fixed_packages) > 0: for fixed_pkg in vuln_matching_fixed_packages: - if fixed_pkg in matching_fixed_packages and command_name( + if fixed_pkg in matching_fixed_packages and univers_version( fixed_pkg.version - ) > command_name(self.version): + ) > univers_version(self.version): later_matching_fixed_packages.append(fixed_pkg) sort_fixed_by_packages_by_version = self.sort_by_version( @@ -788,7 +748,7 @@ def fixed_package_details(self): closest_fixed_package = sort_fixed_by_packages_by_version[0] closest_fixed_package_vulns = closest_fixed_package.affected_by - + # 2023-08-13 Sunday 16:25:41. Not sure but I think I need to define this initially above just after closest_fixed_package = "" closest_fixed_package_vulns_dict = [ { "vuln_id": fixed_pkg_vuln.vulnerability_id, @@ -803,6 +763,7 @@ def fixed_package_details(self): for dict_vuln in purl_dict["vulnerabilities"]: closest_non_vulnerable_fix = "" + # TODO: 2023-08-13 Sunday 16:01:57. Refactor here and elsewhere! if len(later_non_vuln_sibs) > 0: closest_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[0] @@ -810,27 +771,45 @@ def fixed_package_details(self): if len(later_non_vuln_sibs) > 0: most_recent_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[-1] else: - most_recent_non_vulnerable_fix = None + most_recent_non_vulnerable_fix = "" if dict_vuln["vulnerability"] == str(vuln): dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) - dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() - dict_vuln["closest_fixed_by_vulnerabilities"] = closest_fixed_package_vulns_dict + if len(vuln_matching_fixed_packages) > 0: + dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() + closest_fixed_package_vulns_dict = [ + { + "vuln_id": fixed_pkg_vuln.vulnerability_id, + "vuln_get_absolute_url": fixed_pkg_vuln.get_absolute_url(), + } + for fixed_pkg_vuln in closest_fixed_package_vulns + ] + dict_vuln[ + "closest_fixed_by_vulnerabilities" + ] = closest_fixed_package_vulns_dict + else: + dict_vuln["closest_fixed_by_url"] = "" + dict_vuln["closest_fixed_by_vulnerabilities"] = [] purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) - purl_dict[ - "closest_non_vulnerable_fix_url" - ] = closest_non_vulnerable_fix.get_absolute_url() + if len(vuln_matching_fixed_packages) > 0: + purl_dict[ + "closest_non_vulnerable_fix_url" + ] = closest_non_vulnerable_fix.get_absolute_url() + purl_dict[ + "most_recent_non_vulnerable_fix_url" + ] = most_recent_non_vulnerable_fix.get_absolute_url() + else: + purl_dict["closest_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix"] = str( most_recent_non_vulnerable_fix ) - purl_dict[ - "most_recent_non_vulnerable_fix_url" - ] = most_recent_non_vulnerable_fix.get_absolute_url() # Temporary print output during dev/testing: # print("\npurl_dict = {}\n".format(purl_dict)) - # print(json.dumps(purl_dict, indent=4, sort_keys=False)) + print(json.dumps(purl_dict, indent=4, sort_keys=False)) # TODO: Consider whether we want to provide the user with an option to output the dictionary to a file. # pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 24bf0e166..8b3115a19 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -10,13 +10,19 @@ import urllib.parse from datetime import datetime from unittest import TestCase +from unittest import mock import pytest +from django.db import transaction +from django.db.models.query import QuerySet from django.db.utils import IntegrityError from freezegun import freeze_time +from packageurl import PackageURL from univers import versions +from univers.version_range import RANGE_CLASS_BY_SCHEMES from vulnerabilities import models +from vulnerabilities.models import Package class TestVulnerabilityModel(TestCase): @@ -100,10 +106,156 @@ def test_cwe_not_present_in_weaknesses_db(self): @pytest.mark.django_db class TestPackageModel(TestCase): + def setUp(self): + self.vuln1 = models.Vulnerability.objects.create( + summary="test-vuln1", + vulnerability_id="VCID-123", + ) + self.vuln2 = models.Vulnerability.objects.create( + summary="test-vuln2", + vulnerability_id="VCID-456", + ) + + # Create a vuln of its own for the fixed_by_package + self.vuln3 = models.Vulnerability.objects.create( + summary="test-vuln-not-used-anywhere", + vulnerability_id="VCID-000", + ) + + self.vulnerablecode_package = models.Package.objects.create( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-databind", + version="2.13.1", + qualifiers={}, + subpath="", + ) + + self.fixed_by_package = models.Package.objects.create( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-databind", + version="2.13.2", + qualifiers={}, + subpath="", + ) + + self.backport_fixed_by_package = models.Package.objects.create( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-databind", + version="2.12.6.1", + qualifiers={}, + subpath="", + ) + + self.non_vulnerable_package = models.Package.objects.create( + type="maven", + namespace="com.fasterxml.jackson.core", + name="jackson-databind", + version="2.14.0-rc1", + qualifiers={}, + subpath="", + ) + + models.PackageRelatedVulnerability.objects.create( + package=self.vulnerablecode_package, + vulnerability=self.vuln1, + fix=False, + ) + + models.PackageRelatedVulnerability.objects.create( + package=self.vulnerablecode_package, + vulnerability=self.vuln2, + fix=False, + ) + + # Create a fixed_by package for vuln1 + models.PackageRelatedVulnerability.objects.create( + package=self.fixed_by_package, + vulnerability=self.vuln1, + fix=True, + ) + + # Add backport_fixed_by_package as a fixed_by for vuln1 -- but this should be excluded because its version is less than the affected package's version. + models.PackageRelatedVulnerability.objects.create( + package=self.backport_fixed_by_package, + vulnerability=self.vuln1, + fix=True, + ) + + # Create a vuln of its own for the fixed_by_packagefixed_by package for vuln1 + models.PackageRelatedVulnerability.objects.create( + package=self.fixed_by_package, + vulnerability=self.vuln3, + fix=False, + ) + + def test_get_vulnerable_packages(self): + vuln_packages = Package.objects.vulnerable() + assert vuln_packages.count() == 3 + assert vuln_packages.distinct().count() == 2 + + # matching_fixed_packages = vulnerablecode_package.get_fixed_packages(vulnerablecode_package) + # assert vuln_packages.distinct()[0] + + # matching_fixed_packages = vuln_packages.distinct()[0].get_fixed_packages( + # vuln_packages.distinct()[0] + # ) + + first_vulnerable_package = vuln_packages.distinct()[0] + matching_fixed_packages = first_vulnerable_package.get_fixed_packages( + first_vulnerable_package + ) + first_fixed_by_package = matching_fixed_packages[0] + + assert ( + first_vulnerable_package.purl + == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" + ) + assert len(matching_fixed_packages) == 2 + assert ( + first_fixed_by_package.purl + == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1" + ) + + purl_dict = { + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "vulnerabilities": [ + { + "vulnerability": "VCID-123", + "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "closest_fixed_by_url": "/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "closest_fixed_by_vulnerabilities": [ + { + "vuln_id": "VCID-000", + "vuln_get_absolute_url": "/vulnerabilities/VCID-000", + } + ], + }, + { + "vulnerability": "VCID-456", + "closest_fixed_by_purl": "There are no reported fixed packages.", + "closest_fixed_by_url": "", + "closest_fixed_by_vulnerabilities": [], + }, + ], + "closest_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "closest_non_vulnerable_fix_url": "", + "most_recent_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "most_recent_non_vulnerable_fix_url": "", + } + + assert vuln_packages.distinct()[0].fixed_package_details == purl_dict + def test_univers_version_comparisons(self): assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") + deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") + deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") + assert versions.DebianVersion(deb01.version) < versions.DebianVersion(deb02.version) + # pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 is a real PURL in the DB # But we need to replace/delete the "%". Test the error: with pytest.raises(versions.InvalidVersion): @@ -124,21 +276,133 @@ def test_univers_version_comparisons(self): # Use SemverVersion instead as a default fallback version for comparisons. assert versions.SemverVersion("0.9") < versions.SemverVersion("0.10") - deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") - deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") - assert versions.DebianVersion(deb01.version) < versions.DebianVersion(deb02.version) + def test_univers_version_class(self): + gem_version = RANGE_CLASS_BY_SCHEMES["gem"].version_class + assert gem_version == versions.RubygemsVersion + + gem_package = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") + gem_package_version = RANGE_CLASS_BY_SCHEMES[gem_package.type].version_class + assert gem_package_version == versions.RubygemsVersion + + deb_version = RANGE_CLASS_BY_SCHEMES["deb"].version_class + assert deb_version == versions.DebianVersion + + deb_package = models.Package.objects.create(type="deb", name="git", version="2.31.1") + deb_package_version = RANGE_CLASS_BY_SCHEMES[deb_package.type].version_class + assert deb_package_version == versions.DebianVersion + + pypi_version = RANGE_CLASS_BY_SCHEMES["pypi"].version_class + assert pypi_version == versions.PypiVersion + + pypi_package = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") + pypi_package_version = RANGE_CLASS_BY_SCHEMES[pypi_package.type].version_class + assert pypi_package_version == versions.PypiVersion + + def test_sort_by_version(self): + list_to_sort = [ + "pkg:npm/sequelize@3.13.1", + "pkg:npm/sequelize@3.10.1", + "pkg:npm/sequelize@3.40.1", + "pkg:npm/sequelize@3.9.1", + ] + + # Convert list of strings ^ to a list of vulnerablecode Package objects. + vuln_pkg_list = [] + for package in list_to_sort: + purl = PackageURL.from_string(package) + attrs = {k: v for k, v in purl.to_dict().items() if v} + vulnerablecode_package = models.Package.objects.create(**attrs) + vuln_pkg_list.append(vulnerablecode_package) - def test_assign_univers_version(self): - requesting_package = models.Package.objects.create(type="deb", name="git", version="2.30.1") + requesting_package = models.Package.objects.create( + type="npm", + name="sequelize", + version="3.0.0", + ) + + sorted_pkgs = requesting_package.sort_by_version(vuln_pkg_list) + first_sorted_item = sorted_pkgs[0] + + assert sorted_pkgs[0].purl == "pkg:npm/sequelize@3.9.1" + assert sorted_pkgs[-1].purl == "pkg:npm/sequelize@3.40.1" + + def test_string_to_purl_to_dict_to_package(self): + # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., + # a . + + # Convert a PURL string to a PURL. + purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + purl = PackageURL.from_string(purl_string) + + assert type(purl) == PackageURL + assert purl.type == "maven" + assert purl.qualifiers == {} + assert purl.subpath == None + + # Convert the PURL to a dictionary. + # It appears that this step is where the unwanted None values are created for qualifiers and + # subpath when the PURL does not already contain values for those attributes. + purl_to_dict = purl.to_dict() + + assert purl_to_dict == { + "type": "maven", + "namespace": "org.apache.tomcat.embed", + "name": "tomcat-embed-core", + "version": "9.0.31", + "qualifiers": None, + "subpath": None, + } + assert purl_to_dict.get("qualifiers") == None + assert purl_to_dict.get("subpath") == None - deb01 = models.Package.objects.create(type="deb", name="git", version="2.31.1") - command_name_deb01 = requesting_package.assign_univers_version(deb01) - assert command_name_deb01 == versions.DebianVersion + # Convert the dictionary to a VulnerableCode Package, i.e., + # a - pypi01 = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") - command_name_pypi01 = requesting_package.assign_univers_version(pypi01) - assert command_name_pypi01 == versions.PypiVersion + # If subpath is None we get error: django.db.utils.IntegrityError: null value in column + # "subpath" violates not-null constraint -- need to convert value from None to empty string. + # Similar issue with qualifiers, which must be converted from None to {}. - gem01 = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") - command_name_gem01 = requesting_package.assign_univers_version(gem01) - assert command_name_gem01 == versions.SemverVersion + # I've structured the following in this way because trying instead to use + # "with pytest.raises(IntegrityError):" will throw the error + # django.db.transaction.TransactionManagementError: An error occurred in the current + # transaction. You can't execute queries until the end of the 'atomic' block. + + try: + with transaction.atomic(): + vulnerablecode_package = models.Package.objects.create( + type=purl_to_dict.get("type"), + namespace=purl_to_dict.get("namespace"), + name=purl_to_dict.get("name"), + version=purl_to_dict.get("version"), + qualifiers=purl_to_dict.get("qualifiers"), + subpath=purl_to_dict.get("subpath"), + ) + except IntegrityError: + print("\nAs expected, an IntegrityError has occurred.\n") + + # This will avoid the IntegrityError: + if purl_to_dict.get("qualifiers") is None: + purl_to_dict["qualifiers"] = {} + if purl_to_dict.get("subpath") is None: + purl_to_dict["subpath"] = "" + + # Check the qualifiers and subpath values again. + assert purl_to_dict.get("qualifiers") == {} + assert purl_to_dict.get("subpath") == "" + + vulnerablecode_package = models.Package.objects.create( + type=purl_to_dict.get("type"), + namespace=purl_to_dict.get("namespace"), + name=purl_to_dict.get("name"), + version=purl_to_dict.get("version"), + qualifiers=purl_to_dict.get("qualifiers"), + subpath=purl_to_dict.get("subpath"), + ) + + assert type(vulnerablecode_package) == models.Package + assert ( + vulnerablecode_package.purl + == "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + ) + assert vulnerablecode_package.qualifiers == {} + assert vulnerablecode_package.subpath == "" From 9a9401a053c5ad0b8e283f43c8b2b5d587023f9f Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 14 Aug 2023 17:42:05 -0700 Subject: [PATCH 27/53] Move weakness test #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/tests/test_models.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 8b3115a19..9220c8471 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -61,6 +61,12 @@ def test_vulnerability_save_without_vulnerability_id(self): == 1 ) + def test_cwe_not_present_in_weaknesses_db(self): + w1 = models.Weakness.objects.create(name="189") + assert w1.weakness is None + assert w1.name is "" + assert w1.description is "" + # FIXME: The fixture code is duplicated. setUpClass is not working with the pytest mark. @pytest.mark.django_db @@ -196,13 +202,6 @@ def test_get_vulnerable_packages(self): assert vuln_packages.count() == 3 assert vuln_packages.distinct().count() == 2 - # matching_fixed_packages = vulnerablecode_package.get_fixed_packages(vulnerablecode_package) - # assert vuln_packages.distinct()[0] - - # matching_fixed_packages = vuln_packages.distinct()[0].get_fixed_packages( - # vuln_packages.distinct()[0] - # ) - first_vulnerable_package = vuln_packages.distinct()[0] matching_fixed_packages = first_vulnerable_package.get_fixed_packages( first_vulnerable_package From 5c6838c2e644d287f9ce07f48d978902436f3de4 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 14 Aug 2023 20:47:05 -0700 Subject: [PATCH 28/53] Modify UI, update dictionary and tests #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 15 ++++++++ .../templates/package_details.html | 35 ++++++++++++------- vulnerabilities/tests/test_models.py | 4 +++ vulnerablecode/static/css/custom.css | 2 +- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index d0824610b..a0d977b56 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -723,8 +723,11 @@ def fixed_package_details(self): purl_dict.update({"vulnerabilities": []}) purl_dict["closest_non_vulnerable_fix"] = "" + purl_dict["closest_non_vulnerable_fix_version"] = "" purl_dict["closest_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix"] = "" + purl_dict["most_recent_non_vulnerable_fix_version"] = "" purl_dict["most_recent_non_vulnerable_fix_url"] = "" for vuln in qs: @@ -775,7 +778,9 @@ def fixed_package_details(self): if dict_vuln["vulnerability"] == str(vuln): dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) + if len(vuln_matching_fixed_packages) > 0: + dict_vuln["closest_fixed_by_version"] = str(closest_fixed_package.version) dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() closest_fixed_package_vulns_dict = [ { @@ -788,19 +793,29 @@ def fixed_package_details(self): "closest_fixed_by_vulnerabilities" ] = closest_fixed_package_vulns_dict else: + dict_vuln["closest_fixed_by_version"] = "" dict_vuln["closest_fixed_by_url"] = "" dict_vuln["closest_fixed_by_vulnerabilities"] = [] purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + if len(vuln_matching_fixed_packages) > 0: + purl_dict["closest_non_vulnerable_fix_version"] = str( + closest_non_vulnerable_fix.version + ) purl_dict[ "closest_non_vulnerable_fix_url" ] = closest_non_vulnerable_fix.get_absolute_url() + purl_dict["most_recent_non_vulnerable_fix_version"] = str( + most_recent_non_vulnerable_fix.version + ) purl_dict[ "most_recent_non_vulnerable_fix_url" ] = most_recent_non_vulnerable_fix.get_absolute_url() else: + purl_dict["closest_non_vulnerable_fix_version"] = "" purl_dict["closest_non_vulnerable_fix_url"] = "" + purl_dict["most_recent_non_vulnerable_fix_version"] = "" purl_dict["most_recent_non_vulnerable_fix_url"] = "" purl_dict["most_recent_non_vulnerable_fix"] = str( diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 772a2d3f5..da2b68ddb 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -47,18 +47,18 @@ - Closest non-vulnerable purl + Non-vulnerable version - {{ fixed_package_details.closest_non_vulnerable_fix }} + {{ fixed_package_details.closest_non_vulnerable_fix_version }} - Latest non-vulnerable purl + Latest non-vulnerable version - {{ fixed_package_details.most_recent_non_vulnerable_fix }} + {{ fixed_package_details.most_recent_non_vulnerable_fix_version }} @@ -80,7 +80,7 @@ Vulnerability Summary Aliases - Closest fixed by packages + Fixed by packages @@ -111,10 +111,24 @@ {% for abc in value %} {% if abc.vulnerability == vulnerability.vulnerability_id %} - {{ abc.closest_fixed_by_purl }} + {% if abc.closest_fixed_by_purl == "There are no reported fixed packages." %} + There are no reported fixed packages. + {% else %} + {{ abc.closest_fixed_by_purl }} + {% endif %}
    - Vulnerabilities: {{ abc.closest_fixed_by_vulnerabilities|length }} + {% if abc.closest_fixed_by_purl == "There are no reported fixed packages." %} + + {% else %} + + {% if abc.closest_fixed_by_vulnerabilities|length != 1 %} + Affected by {{ abc.closest_fixed_by_vulnerabilities|length }} other vulnerabilities. + {% else %} + Affected by {{ abc.closest_fixed_by_vulnerabilities|length }} other vulnerability. + {% endif %} + + {% endif %} {% if abc.closest_fixed_by_vulnerabilities|length != 0 %}
    @@ -125,12 +139,7 @@
    - The closest fixed-by purl has {{ abc.closest_fixed_by_vulnerabilities|length }} - {% if abc.closest_fixed_by_vulnerabilities|length == 1 %} - vulnerability: - {% else %} - vulnerabilities: - {% endif %} + This version is affected by these other vulnerabilities:
    {% for closest_vuln in abc.closest_fixed_by_vulnerabilities %}
    diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 9220c8471..f8e85c9df 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -224,6 +224,7 @@ def test_get_vulnerable_packages(self): { "vulnerability": "VCID-123", "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "closest_fixed_by_version": "2.13.2", "closest_fixed_by_url": "/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", "closest_fixed_by_vulnerabilities": [ { @@ -235,13 +236,16 @@ def test_get_vulnerable_packages(self): { "vulnerability": "VCID-456", "closest_fixed_by_purl": "There are no reported fixed packages.", + "closest_fixed_by_version": "", "closest_fixed_by_url": "", "closest_fixed_by_vulnerabilities": [], }, ], "closest_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "closest_non_vulnerable_fix_version": "", "closest_non_vulnerable_fix_url": "", "most_recent_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", + "most_recent_non_vulnerable_fix_version": "", "most_recent_non_vulnerable_fix_url": "", } diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index bf094f70c..282228560 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -196,7 +196,7 @@ code { } .two-col-left { - width: 250px; + width: 255px; text-align: right !important; font-weight: bold; padding-right: 15px !important; From 3ef542b505ddeb5f3f4da2240d7cc2bea2380474 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 28 Aug 2023 09:31:27 -0700 Subject: [PATCH 29/53] Begin replacing strings with objects in package details dictionary #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 298 +++++++++++++----- .../templates/package_details.html | 227 +++++++++++-- vulnerabilities/tests/test_models.py | 32 +- vulnerablecode/settings.py | 1 + 4 files changed, 427 insertions(+), 131 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index a0d977b56..4caa200c4 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -663,6 +663,7 @@ def get_sibling_packages(self, package): """ Return a queryset of all packages with the same type, namespace, name, subpath and qualifiers as the target package, whether or not the 'sibling packages' fix any vulnerability. + Identical to get_fixed_packages() above except for the removal of 'packagerelatedvulnerability__fix=True,'. """ return Package.objects.filter( name=package.name, @@ -673,16 +674,13 @@ def get_sibling_packages(self, package): ).distinct() def sort_by_version(self, packages_to_sort): - # Incoming is a list of + # Incoming is a list of objects. univers_version = RANGE_CLASS_BY_SCHEMES[packages_to_sort[0].type].version_class - sorted_by_version = [] - sorted_by_version = sorted( + return sorted( packages_to_sort, key=lambda x: univers_version(x.version), ) - return sorted_by_version - @property def fixed_package_details(self): """ @@ -692,7 +690,7 @@ def fixed_package_details(self): # Get all fixed packages that match the target package (type etc.), regardless of fixed vuln. matching_fixed_packages = self.get_fixed_packages(package=self) - # Get a list of the vulnerabilities that affect this package (i.e., self). + # Get a list of the vulnerabilities that affect the current package (i.e., self). qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) all_sibling_packages = self.get_sibling_packages(package=self) @@ -702,14 +700,17 @@ def fixed_package_details(self): if sib.is_vulnerable is False: non_vuln_sibs.append(sib) - # Add just the greater-than versions to a new list + # Add the greater-than versions to a new list. univers_version = RANGE_CLASS_BY_SCHEMES[self.type].version_class later_non_vuln_sibs = [] for non_vuln_sib in non_vuln_sibs: if univers_version(non_vuln_sib.version) > univers_version(self.version): later_non_vuln_sibs.append(non_vuln_sib) - # Take the list of vulns affecting the current package, retrieve a list of the fixed packages for each vuln, and assign the result to a custom attribute, 'matching_fixed_packages'. Ex: qs[0].matching_fixed_packages gives us the fixed package(s) for the 1st vuln for this affected package (i.e., self). + # Take the list of vulnerabilities affecting the current package, retrieve a list of the fixed + # packages for each, and assign the result to a custom attribute, 'matching_fixed_packages'. + # Ex: qs[0].matching_fixed_packages returns the fixed package(s) for the 1st vulnerability + # for this affected package (i.e., self). qs = qs.prefetch_related( Prefetch( "packages", @@ -719,24 +720,34 @@ def fixed_package_details(self): ) purl_dict = {} - purl_dict["purl"] = self.purl - purl_dict.update({"vulnerabilities": []}) - - purl_dict["closest_non_vulnerable_fix"] = "" - purl_dict["closest_non_vulnerable_fix_version"] = "" - purl_dict["closest_non_vulnerable_fix_url"] = "" - - purl_dict["most_recent_non_vulnerable_fix"] = "" - purl_dict["most_recent_non_vulnerable_fix_version"] = "" - purl_dict["most_recent_non_vulnerable_fix_url"] = "" + # 2023-08-23 Wednesday 13:51:17. Let's remove this next. + # purl_dict["purl"] = self.purl + # Pass a PackageURL object to the Jinja2 template via the context dictionary. + purl_dict["test_purl"] = PackageURL.from_string(self.purl) + # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". + # purl_dict.update({"vulnerabilities": []}) + purl_dict.update({"test_vulnerabilities": []}) + + # # ID when there are 0 vulnerabilities. + # print("\nqs = {}\n".format(qs)) + # print("\nlen(qs) = {}\n".format(len(qs))) for vuln in qs: later_matching_fixed_packages = [] - purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". + # purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) + # Pass a Vulnerability object. + purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) + # v1 = vuln + # print("\nv1 = {}\n".format(v1)) + # print("\ntype(v1) = {}\n".format(type(v1))) + # print("\nv1.severities = {}\n".format(v1.severities)) + # print("\nv1.affected_packages = {}\n".format(v1.affected_packages)) + vuln_matching_fixed_packages = vuln.matching_fixed_packages closest_fixed_package = "" - # Need to define here to avoid "" vs. [] for some closest_fixed_by_vulnerabilities values? - closest_fixed_package_vulns_dict = [] + # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. + # closest_fixed_package_vulns_dict = [] if len(vuln_matching_fixed_packages) > 0: for fixed_pkg in vuln_matching_fixed_packages: @@ -745,91 +756,216 @@ def fixed_package_details(self): ) > univers_version(self.version): later_matching_fixed_packages.append(fixed_pkg) - sort_fixed_by_packages_by_version = self.sort_by_version( - later_matching_fixed_packages - ) + # We already identified a closest fixed package and are looking for a later fixed package. + if later_matching_fixed_packages: + sort_fixed_by_packages_by_version = self.sort_by_version( + later_matching_fixed_packages + ) + # closest_fixed_package is a 'vulnerabilities.models.Package' closest_fixed_package = sort_fixed_by_packages_by_version[0] + closest_fixed_package_vulns = closest_fixed_package.affected_by - # 2023-08-13 Sunday 16:25:41. Not sure but I think I need to define this initially above just after closest_fixed_package = "" - closest_fixed_package_vulns_dict = [ - { - "vuln_id": fixed_pkg_vuln.vulnerability_id, - "vuln_get_absolute_url": fixed_pkg_vuln.get_absolute_url(), - } - for fixed_pkg_vuln in closest_fixed_package_vulns - ] + # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. + # closest_fixed_package_vulns_dict = [ + # { + # "vuln_id": fixed_pkg_vuln.vulnerability_id, + # } + # for fixed_pkg_vuln in closest_fixed_package_vulns + # ] else: closest_fixed_package = "There are no reported fixed packages." - for dict_vuln in purl_dict["vulnerabilities"]: - closest_non_vulnerable_fix = "" - - # TODO: 2023-08-13 Sunday 16:01:57. Refactor here and elsewhere! + # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". + # for dict_vuln in purl_dict["vulnerabilities"]: + # # for dict_vuln in purl_dict["test_vulnerabilities"]: + # closest_non_vulnerable_sib = "" + # if len(later_non_vuln_sibs) > 0: + # closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] + # else: + # closest_non_vulnerable_sib = "" + + # # most_recent_non_vulnerable_sib = "" + # latest_non_vulnerable_sib = "" + # if len(later_non_vuln_sibs) > 0: + # latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] + # else: + # latest_non_vulnerable_sib = "" + + # if dict_vuln["vulnerability"] == str(vuln): + # # ALERT: 2023-08-22 Tuesday 18:25:42. Try this. 2023-08-22 Tuesday 18:27:13. No does not work. + # # if dict_vuln["test_vulnerability"] == str(vuln): + + # if len(vuln_matching_fixed_packages) > 0: + # # dict_vuln["closest_fixed_by"] = purl_to_dict_with_purl( + # # closest_fixed_package.purl + # # ) + + # dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( + # closest_fixed_package.purl + # ) + # # print("For test_fixed_by_purl:") + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl) = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl) + # # ) + # # ) + # # print( + # # "\ntype(PackageURL.from_string(closest_fixed_package.purl)) = {}\n".format( + # # type(PackageURL.from_string(closest_fixed_package.purl)) + # # ) + # # ) + # # print( + # # "\nclosest_fixed_package.purl = {}\n".format(closest_fixed_package.purl) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).type = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).type + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).namespace = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).namespace + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).name = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).name + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).version = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).version + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).qualifiers = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).qualifiers + # # ) + # # ) + # # print( + # # "\nPackageURL.from_string(closest_fixed_package.purl).subpath = {}\n".format( + # # PackageURL.from_string(closest_fixed_package.purl).subpath + # # ) + # # ) + + # # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. + # # closest_fixed_package_vulns_dict = [ + # # { + # # "vuln_id": fixed_pkg_vuln.vulnerability_id, + # # } + # # for fixed_pkg_vuln in closest_fixed_package_vulns + # # ] + # # dict_vuln[ + # # "closest_fixed_by_vulnerabilities" + # # ] = closest_fixed_package_vulns_dict + + # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ + # fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns + # ] + # else: + # # dict_vuln["closest_fixed_by"] = {} + # # dict_vuln["closest_fixed_by_vulnerabilities"] = [] + + # dict_vuln["test_fixed_by_purl"] = None + # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] + + # if len(vuln_matching_fixed_packages) > 0: + # # purl_dict["closest_non_vulnerable"] = purl_to_dict_with_purl( + # # closest_non_vulnerable_sib.purl + # # ) + # # purl_dict["latest_non_vulnerable"] = purl_to_dict_with_purl( + # # latest_non_vulnerable_sib.purl + # # ) + + # purl_dict["test_closest_non_vulnerable"] = PackageURL.from_string( + # closest_non_vulnerable_sib.purl + # ) + # purl_dict["test_latest_non_vulnerable"] = PackageURL.from_string( + # latest_non_vulnerable_sib.purl + # ) + + # # purl_dict["most_recent_non_vulnerable_fix_version"] = str( + # # most_recent_non_vulnerable_fix.version + # # ) + # # purl_dict[ + # # "most_recent_non_vulnerable_fix_url" + # # ] = most_recent_non_vulnerable_fix.get_absolute_url() + # else: + # # purl_dict["closest_non_vulnerable"] = {} + # # purl_dict["latest_non_vulnerable"] = {} + + # # purl_dict["test_closest_non_vulnerable"] = "" + # # purl_dict["test_latest_non_vulnerable"] = "" + + # purl_dict["test_closest_non_vulnerable"] = None + # purl_dict["test_latest_non_vulnerable"] = None + + # ====================================================== + # TODO: 2023-08-22 Tuesday 18:37:15. Adapt this new section, from above, to the "test_vulnerabilities" approach, changing names as needed. + # for dict_vuln in purl_dict["vulnerabilities"]: + for dict_vuln in purl_dict["test_vulnerabilities"]: + closest_non_vulnerable_sib = "" if len(later_non_vuln_sibs) > 0: - closest_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[0] + closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] + else: + closest_non_vulnerable_sib = "" - most_recent_non_vulnerable_fix = "" + latest_non_vulnerable_sib = "" if len(later_non_vuln_sibs) > 0: - most_recent_non_vulnerable_fix = self.sort_by_version(later_non_vuln_sibs)[-1] + latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] else: - most_recent_non_vulnerable_fix = "" + latest_non_vulnerable_sib = "" - if dict_vuln["vulnerability"] == str(vuln): - dict_vuln["closest_fixed_by_purl"] = str(closest_fixed_package) + # if dict_vuln["vulnerability"] == str(vuln): + if dict_vuln["test_vulnerability"] == vuln: if len(vuln_matching_fixed_packages) > 0: - dict_vuln["closest_fixed_by_version"] = str(closest_fixed_package.version) - dict_vuln["closest_fixed_by_url"] = closest_fixed_package.get_absolute_url() - closest_fixed_package_vulns_dict = [ - { - "vuln_id": fixed_pkg_vuln.vulnerability_id, - "vuln_get_absolute_url": fixed_pkg_vuln.get_absolute_url(), - } - for fixed_pkg_vuln in closest_fixed_package_vulns + + dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( + closest_fixed_package.purl + ) + + # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. + # closest_fixed_package_vulns_dict = [ + # { + # "vuln_id": fixed_pkg_vuln.vulnerability_id, + # } + # for fixed_pkg_vuln in closest_fixed_package_vulns + # ] + + dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ + fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] - dict_vuln[ - "closest_fixed_by_vulnerabilities" - ] = closest_fixed_package_vulns_dict else: - dict_vuln["closest_fixed_by_version"] = "" - dict_vuln["closest_fixed_by_url"] = "" - dict_vuln["closest_fixed_by_vulnerabilities"] = [] - purl_dict["closest_non_vulnerable_fix"] = str(closest_non_vulnerable_fix) + dict_vuln["test_fixed_by_purl"] = None + dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] if len(vuln_matching_fixed_packages) > 0: - purl_dict["closest_non_vulnerable_fix_version"] = str( - closest_non_vulnerable_fix.version + + purl_dict["TEST_test_closest_non_vulnerable"] = PackageURL.from_string( + closest_non_vulnerable_sib.purl ) - purl_dict[ - "closest_non_vulnerable_fix_url" - ] = closest_non_vulnerable_fix.get_absolute_url() - purl_dict["most_recent_non_vulnerable_fix_version"] = str( - most_recent_non_vulnerable_fix.version + purl_dict["TEST_test_latest_non_vulnerable"] = PackageURL.from_string( + latest_non_vulnerable_sib.purl ) - purl_dict[ - "most_recent_non_vulnerable_fix_url" - ] = most_recent_non_vulnerable_fix.get_absolute_url() + else: - purl_dict["closest_non_vulnerable_fix_version"] = "" - purl_dict["closest_non_vulnerable_fix_url"] = "" - purl_dict["most_recent_non_vulnerable_fix_version"] = "" - purl_dict["most_recent_non_vulnerable_fix_url"] = "" - purl_dict["most_recent_non_vulnerable_fix"] = str( - most_recent_non_vulnerable_fix - ) + purl_dict["TEST_test_closest_non_vulnerable"] = None + purl_dict["TEST_test_latest_non_vulnerable"] = None + # ====================================================== - # Temporary print output during dev/testing: - # print("\npurl_dict = {}\n".format(purl_dict)) - print(json.dumps(purl_dict, indent=4, sort_keys=False)) + # Temporary print output during dev/testing. + from pprint import pprint - # TODO: Consider whether we want to provide the user with an option to output the dictionary to a file. - # pretty_purl_dict = json.dumps(purl_dict, indent=4, sort_keys=False) - # with open("pretty_purl_dict.txt", "w") as f: - # f.write(pretty_purl_dict) + pprint( + purl_dict, + sort_dicts=False, + ) + print("") return purl_dict diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index da2b68ddb..60d08f971 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -33,7 +33,10 @@ - {{ package.purl }} + + {{ fixed_package_details.test_purl.to_string }} @@ -50,7 +53,24 @@ Non-vulnerable version - {{ fixed_package_details.closest_non_vulnerable_fix_version }} + Non-vulnerable version + + {{ fixed_package_details.TEST_test_closest_non_vulnerable.version }} + + + + + + + Non-vulnerable version
    (non-breaking) + + + [to come] @@ -58,7 +78,22 @@ Latest non-vulnerable version - {{ fixed_package_details.most_recent_non_vulnerable_fix_version }} + Latest non-vulnerable version + + {{ fixed_package_details.TEST_test_latest_non_vulnerable.version }} + + + + + Latest non-vulnerable version
    (non-breaking) + + + [to come] @@ -69,7 +104,7 @@
    - Affected by vulnerabilities ({{ affected_by_vulnerabilities|length }}) + Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }})
    @@ -77,10 +112,9 @@ - + - - + @@ -89,11 +123,9 @@ - - + {% empty %} - @@ -175,13 +338,13 @@
    - Fixed by vulnerabilities ({{ fixing_vulnerabilities|length }}) + Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }})
    VulnerabilityVulnerability SummaryAliasesFixed by packagesFixed by
    {{ vulnerability.vulnerability_id }} - - {{ vulnerability.summary }} - +
    + Aliases: +
    {% for alias in vulnerability.alias %} {% if alias.url %} {{ alias }} @@ -104,36 +136,83 @@ {% endif %} {% endfor %}     		+ {{ vulnerability.summary }} + - {% if package.purl in fixed_package_details.purl %} + {# if package.purl in fixed_package_details.purl #} + {# if package.purl == fixed_package_details.test_purl.purl #} + {% if package.purl == fixed_package_details.test_purl.to_string %} + package.purl = {{ package.purl }} +
    + fixed_package_details.purl = {{ fixed_package_details.purl }} +
    + yikes +
    + fixed_package_details.test_purl = {{ fixed_package_details.test_purl }} +
    + fixed_package_details.test_purl.to_string = {{ fixed_package_details.test_purl.to_string }} +
    + {% for key, value in fixed_package_details.items %} - {% if key == "vulnerabilities" %} - {% for abc in value %} - {% if abc.vulnerability == vulnerability.vulnerability_id %} + {# if key == "vulnerabilities" #} + {% if key == "test_vulnerabilities" %} + goodby + {% for vuln in value %} +
    + vulnerability = {{ vulnerability }} +
    + vuln = {{ vuln }} +
    + vuln.test_vulnerability = {{ vuln.test_vulnerability }} +
    + vuln.test_vulnerability.to_string = {{ vuln.test_vulnerability.to_string }} +
    + vuln.test_vulnerability.vulnerability_id = {{ vuln.test_vulnerability.vulnerability_id }} +
    + vulnerability.vulnerability_id = {{ vulnerability.vulnerability_id }} + + {# if vuln.vulnerability == vulnerability.vulnerability_id #} + {% if vuln.test_vulnerability.vulnerability_id == vulnerability.vulnerability_id %} +
    + hello - {% if abc.closest_fixed_by_purl == "There are no reported fixed packages." %} - There are no reported fixed packages. + + + - {% if abc.closest_fixed_by_purl == "There are no reported fixed packages." %} + - {% if abc.closest_fixed_by_vulnerabilities|length != 0 %} + + + {% if vuln.test_fixed_by_purl is None %} + + + +
    + + There are no reported fixed versions. + +
    + + {% else %} + + {% if vuln.test_fixed_by_purl_vulnerabilities|length == 0 %} + + +
    + + {{ vuln.test_fixed_by_purl.version }} + +
    + Affected by 0 other vulnerabilities. + +
    + + {% else %} + + + +
    + + {{ vuln.test_fixed_by_purl.version }} + + {% if vuln.test_fixed_by_purl_vulnerabilities|length != 1 %} +
    + Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + {% else %} +
    + Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerability. + {% endif %} + +
    +
    + +
    + +
    + +
    + + {% endif %} + {% endif %} + {% endif %} {% endfor %} {% endif %} @@ -164,7 +327,7 @@
    + This package is not known to be affected by vulnerabilities.
    - + diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index f8e85c9df..9605452e4 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -223,30 +223,25 @@ def test_get_vulnerable_packages(self): "vulnerabilities": [ { "vulnerability": "VCID-123", - "closest_fixed_by_purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - "closest_fixed_by_version": "2.13.2", - "closest_fixed_by_url": "/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - "closest_fixed_by_vulnerabilities": [ - { - "vuln_id": "VCID-000", - "vuln_get_absolute_url": "/vulnerabilities/VCID-000", - } - ], + "closest_fixed_by": { + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "type": "maven", + "namespace": "com.fasterxml.jackson.core", + "name": "jackson-databind", + "version": "2.13.2", + "qualifiers": {}, + "subpath": "", + }, + "closest_fixed_by_vulnerabilities": [{"vuln_id": "VCID-000"}], }, { "vulnerability": "VCID-456", - "closest_fixed_by_purl": "There are no reported fixed packages.", - "closest_fixed_by_version": "", - "closest_fixed_by_url": "", + "closest_fixed_by": {}, "closest_fixed_by_vulnerabilities": [], }, ], - "closest_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "closest_non_vulnerable_fix_version": "", - "closest_non_vulnerable_fix_url": "", - "most_recent_non_vulnerable_fix": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1", - "most_recent_non_vulnerable_fix_version": "", - "most_recent_non_vulnerable_fix_url": "", + "closest_non_vulnerable": {}, + "latest_non_vulnerable": {}, } assert vuln_packages.distinct()[0].fixed_package_details == purl_dict @@ -343,6 +338,7 @@ def test_string_to_purl_to_dict_to_package(self): assert purl.subpath == None # Convert the PURL to a dictionary. + # ALERT: 2023-08-15 Tuesday 13:18:09. What about using the function 'def purl_to_dict(purl: PackageURL)'? Confusingly similar name but it seems designed to address the issue raised here (and looks useful for passing the data to the Jinja2 template). # It appears that this step is where the unwanted None values are created for qualifiers and # subpath when the PURL does not already contain values for those attributes. purl_to_dict = purl.to_dict() diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 525127915..99e72c15f 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -40,6 +40,7 @@ # SECURITY WARNING: do not run with debug turned on in production DEBUG = env.bool("VULNERABLECODE_DEBUG", default=False) +DEBUG = True # SECURITY WARNING: do not run with debug turned on in production DEBUG_TOOLBAR = env.bool("VULNERABLECODE_DEBUG_TOOLBAR", default=False) From 1045b3618fb0eb853f2091ca0d38327fc0c11f37 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 28 Aug 2023 17:04:14 -0700 Subject: [PATCH 30/53] Clean current package details template and related model code #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 163 ------------ .../templates/package_details.html | 247 ++++-------------- 2 files changed, 46 insertions(+), 364 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 4caa200c4..b52268493 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -720,34 +720,17 @@ def fixed_package_details(self): ) purl_dict = {} - # 2023-08-23 Wednesday 13:51:17. Let's remove this next. - # purl_dict["purl"] = self.purl # Pass a PackageURL object to the Jinja2 template via the context dictionary. purl_dict["test_purl"] = PackageURL.from_string(self.purl) - # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". - # purl_dict.update({"vulnerabilities": []}) purl_dict.update({"test_vulnerabilities": []}) - # # ID when there are 0 vulnerabilities. - # print("\nqs = {}\n".format(qs)) - # print("\nlen(qs) = {}\n".format(len(qs))) - for vuln in qs: later_matching_fixed_packages = [] - # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". - # purl_dict["vulnerabilities"].append({"vulnerability": vuln.vulnerability_id}) # Pass a Vulnerability object. purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) - # v1 = vuln - # print("\nv1 = {}\n".format(v1)) - # print("\ntype(v1) = {}\n".format(type(v1))) - # print("\nv1.severities = {}\n".format(v1.severities)) - # print("\nv1.affected_packages = {}\n".format(v1.affected_packages)) vuln_matching_fixed_packages = vuln.matching_fixed_packages closest_fixed_package = "" - # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. - # closest_fixed_package_vulns_dict = [] if len(vuln_matching_fixed_packages) > 0: for fixed_pkg in vuln_matching_fixed_packages: @@ -762,149 +745,13 @@ def fixed_package_details(self): later_matching_fixed_packages ) - # closest_fixed_package is a 'vulnerabilities.models.Package' closest_fixed_package = sort_fixed_by_packages_by_version[0] closest_fixed_package_vulns = closest_fixed_package.affected_by - # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. - # closest_fixed_package_vulns_dict = [ - # { - # "vuln_id": fixed_pkg_vuln.vulnerability_id, - # } - # for fixed_pkg_vuln in closest_fixed_package_vulns - # ] else: closest_fixed_package = "There are no reported fixed packages." - # 2023-08-23 Wednesday 15:16:26. Time to remove "vulnerabilities". - # for dict_vuln in purl_dict["vulnerabilities"]: - # # for dict_vuln in purl_dict["test_vulnerabilities"]: - # closest_non_vulnerable_sib = "" - # if len(later_non_vuln_sibs) > 0: - # closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - # else: - # closest_non_vulnerable_sib = "" - - # # most_recent_non_vulnerable_sib = "" - # latest_non_vulnerable_sib = "" - # if len(later_non_vuln_sibs) > 0: - # latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] - # else: - # latest_non_vulnerable_sib = "" - - # if dict_vuln["vulnerability"] == str(vuln): - # # ALERT: 2023-08-22 Tuesday 18:25:42. Try this. 2023-08-22 Tuesday 18:27:13. No does not work. - # # if dict_vuln["test_vulnerability"] == str(vuln): - - # if len(vuln_matching_fixed_packages) > 0: - # # dict_vuln["closest_fixed_by"] = purl_to_dict_with_purl( - # # closest_fixed_package.purl - # # ) - - # dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( - # closest_fixed_package.purl - # ) - # # print("For test_fixed_by_purl:") - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl) = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl) - # # ) - # # ) - # # print( - # # "\ntype(PackageURL.from_string(closest_fixed_package.purl)) = {}\n".format( - # # type(PackageURL.from_string(closest_fixed_package.purl)) - # # ) - # # ) - # # print( - # # "\nclosest_fixed_package.purl = {}\n".format(closest_fixed_package.purl) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).type = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).type - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).namespace = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).namespace - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).name = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).name - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).version = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).version - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).qualifiers = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).qualifiers - # # ) - # # ) - # # print( - # # "\nPackageURL.from_string(closest_fixed_package.purl).subpath = {}\n".format( - # # PackageURL.from_string(closest_fixed_package.purl).subpath - # # ) - # # ) - - # # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. - # # closest_fixed_package_vulns_dict = [ - # # { - # # "vuln_id": fixed_pkg_vuln.vulnerability_id, - # # } - # # for fixed_pkg_vuln in closest_fixed_package_vulns - # # ] - # # dict_vuln[ - # # "closest_fixed_by_vulnerabilities" - # # ] = closest_fixed_package_vulns_dict - - # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ - # fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns - # ] - # else: - # # dict_vuln["closest_fixed_by"] = {} - # # dict_vuln["closest_fixed_by_vulnerabilities"] = [] - - # dict_vuln["test_fixed_by_purl"] = None - # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] - - # if len(vuln_matching_fixed_packages) > 0: - # # purl_dict["closest_non_vulnerable"] = purl_to_dict_with_purl( - # # closest_non_vulnerable_sib.purl - # # ) - # # purl_dict["latest_non_vulnerable"] = purl_to_dict_with_purl( - # # latest_non_vulnerable_sib.purl - # # ) - - # purl_dict["test_closest_non_vulnerable"] = PackageURL.from_string( - # closest_non_vulnerable_sib.purl - # ) - # purl_dict["test_latest_non_vulnerable"] = PackageURL.from_string( - # latest_non_vulnerable_sib.purl - # ) - - # # purl_dict["most_recent_non_vulnerable_fix_version"] = str( - # # most_recent_non_vulnerable_fix.version - # # ) - # # purl_dict[ - # # "most_recent_non_vulnerable_fix_url" - # # ] = most_recent_non_vulnerable_fix.get_absolute_url() - # else: - # # purl_dict["closest_non_vulnerable"] = {} - # # purl_dict["latest_non_vulnerable"] = {} - - # # purl_dict["test_closest_non_vulnerable"] = "" - # # purl_dict["test_latest_non_vulnerable"] = "" - - # purl_dict["test_closest_non_vulnerable"] = None - # purl_dict["test_latest_non_vulnerable"] = None - - # ====================================================== - # TODO: 2023-08-22 Tuesday 18:37:15. Adapt this new section, from above, to the "test_vulnerabilities" approach, changing names as needed. - # for dict_vuln in purl_dict["vulnerabilities"]: for dict_vuln in purl_dict["test_vulnerabilities"]: closest_non_vulnerable_sib = "" if len(later_non_vuln_sibs) > 0: @@ -918,7 +765,6 @@ def fixed_package_details(self): else: latest_non_vulnerable_sib = "" - # if dict_vuln["vulnerability"] == str(vuln): if dict_vuln["test_vulnerability"] == vuln: if len(vuln_matching_fixed_packages) > 0: @@ -927,14 +773,6 @@ def fixed_package_details(self): closest_fixed_package.purl ) - # 2023-08-23 Wednesday 13:45:12. Apparently no longer used. - # closest_fixed_package_vulns_dict = [ - # { - # "vuln_id": fixed_pkg_vuln.vulnerability_id, - # } - # for fixed_pkg_vuln in closest_fixed_package_vulns - # ] - dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] @@ -956,7 +794,6 @@ def fixed_package_details(self): purl_dict["TEST_test_closest_non_vulnerable"] = None purl_dict["TEST_test_latest_non_vulnerable"] = None - # ====================================================== # Temporary print output during dev/testing. from pprint import pprint diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 60d08f971..d24a7ab40 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -33,9 +33,6 @@ @@ -45,60 +42,44 @@ {% if affected_by_vulnerabilities|length != 0 %} -
    -
    VulnerabilityVulnerability Summary Aliases
    - {{ fixed_package_details.test_purl.to_string }}
    - - - - - - - - - - - - - - - - - - -
    - Non-vulnerable version - - Non-vulnerable version - - {{ fixed_package_details.TEST_test_closest_non_vulnerable.version }} - - -
    - Non-vulnerable version
    (non-breaking) -     		- [to come] -
    - Latest non-vulnerable version - - Latest non-vulnerable version - - {{ fixed_package_details.TEST_test_latest_non_vulnerable.version }} -
    - Latest non-vulnerable version
    (non-breaking) -     		- [to come] -
    -
    +
    + + + + + + + + + + + + + + + + + + + +
    + Non-vulnerable version + + {{ fixed_package_details.TEST_test_closest_non_vulnerable.version }} +
    + Non-vulnerable version
    (non-breaking) +     		+ [to come] +
    + Latest non-vulnerable version + + {{ fixed_package_details.TEST_test_latest_non_vulnerable.version }} +
    + Latest non-vulnerable version
    (non-breaking) +     		+ [to come] +
    +
    {% endif %} @@ -107,8 +88,6 @@ Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }})
    - - @@ -140,154 +119,26 @@ {{ vulnerability.summary }} @@ -385,5 +231,4 @@ {% endif %} - {% endblock %} From 57007b9194c68cca4822e810750c5205465066a3 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Wed, 30 Aug 2023 12:20:12 -0700 Subject: [PATCH 31/53] Begin work on major-version issue #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 128 +++++++++++++++--- .../templates/package_details.html | 38 +++--- vulnerabilities/tests/test_models.py | 110 +++++++++++++++ 3 files changed, 239 insertions(+), 37 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index b52268493..716fdc33a 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -702,6 +702,7 @@ def fixed_package_details(self): # Add the greater-than versions to a new list. univers_version = RANGE_CLASS_BY_SCHEMES[self.type].version_class + print("\nunivers_version = {}".format(univers_version)) later_non_vuln_sibs = [] for non_vuln_sib in non_vuln_sibs: if univers_version(non_vuln_sib.version) > univers_version(self.version): @@ -721,13 +722,34 @@ def fixed_package_details(self): purl_dict = {} # Pass a PackageURL object to the Jinja2 template via the context dictionary. - purl_dict["test_purl"] = PackageURL.from_string(self.purl) - purl_dict.update({"test_vulnerabilities": []}) + # purl_dict["test_purl"] = PackageURL.from_string(self.purl) + # purl_dict.update({"test_vulnerabilities": []}) + purl_dict["purl"] = PackageURL.from_string(self.purl) + print("\npurl = {}".format(purl_dict["purl"])) + + purl_univers_version = univers_version(purl_dict["purl"].version) + print("\npurl_univers_version = {}".format(purl_univers_version)) + + # ALERT: This throws an error: AttributeError: 'MavenVersion' object has no attribute 'major' + # How do we compare major versions? + # purl_univers_version_major = purl_univers_version.major + + # purl_semver_version = versions.SemverVersion(purl_dict["purl"].version) + # print("\npurl_semver_version = {}".format(purl_semver_version)) + + # purl_semver_version_major = purl_semver_version.major + # print("\npurl_semver_version_major = {}".format(purl_semver_version_major)) + + purl_semver_version_major = versions.SemverVersion(purl_dict["purl"].version).major + print("\npurl_semver_version_major = {}\n".format(purl_semver_version_major)) + + purl_dict.update({"vulnerabilities": []}) for vuln in qs: later_matching_fixed_packages = [] # Pass a Vulnerability object. - purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) + # purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) + purl_dict["vulnerabilities"].append({"vulnerability": vuln}) vuln_matching_fixed_packages = vuln.matching_fixed_packages closest_fixed_package = "" @@ -744,6 +766,29 @@ def fixed_package_details(self): sort_fixed_by_packages_by_version = self.sort_by_version( later_matching_fixed_packages ) + print( + "\nsort_fixed_by_packages_by_version = {}\n".format( + sort_fixed_by_packages_by_version + ) + ) + for fixed_by_pkg in sort_fixed_by_packages_by_version: + print("\nfixed_by_pkg.version = {}\n".format(fixed_by_pkg.version)) + purl_from_fixed_by_pkg = PackageURL.from_string(fixed_by_pkg.purl) + print("\npurl_from_fixed_by_pkg = {}\n".format(purl_from_fixed_by_pkg)) + + print("\nfixed_by_pkg = {}\n".format(fixed_by_pkg)) + + # purl_semver_version_major = versions.SemverVersion(purl_dict["purl"].version).major + # print("\npurl_semver_version_major = {}\n".format(purl_semver_version_major)) + + fixed_by_pkg_semver_version_major = versions.SemverVersion( + fixed_by_pkg.version + ).major + print( + "\nfixed_by_pkg_semver_version_major = {}\n".format( + fixed_by_pkg_semver_version_major + ) + ) closest_fixed_package = sort_fixed_by_packages_by_version[0] @@ -752,48 +797,95 @@ def fixed_package_details(self): else: closest_fixed_package = "There are no reported fixed packages." - for dict_vuln in purl_dict["test_vulnerabilities"]: + # for dict_vuln in purl_dict["test_vulnerabilities"]: + for dict_vuln in purl_dict["vulnerabilities"]: closest_non_vulnerable_sib = "" - if len(later_non_vuln_sibs) > 0: - closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - else: - closest_non_vulnerable_sib = "" + # if len(later_non_vuln_sibs) > 0: + # closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] + # else: + # closest_non_vulnerable_sib = "" latest_non_vulnerable_sib = "" + # if len(later_non_vuln_sibs) > 0: + # latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] + # else: + # latest_non_vulnerable_sib = "" + + closest_non_vulnerable_sib_non_breaking = "" + latest_non_vulnerable_sib_non_breaking = "" + if len(later_non_vuln_sibs) > 0: + closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] + + print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) + + later_non_vuln_sibs_non_breaking = [] + for non_vuln_sib in later_non_vuln_sibs: + print("\nnon_vuln_sib = {}\n".format(non_vuln_sib)) + print("\ntype(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) + + # 2023-08-28 Monday 19:19:03. I need to resume here and populate the list later_non_vuln_sibs_non_breaking, then see whether there are any non-breaking versions and in anmy event add 2 more dict entries for the closest non-breaking and latest non-breaking versions. else: + closest_non_vulnerable_sib = "" latest_non_vulnerable_sib = "" + # FIXME: 2023-08-30 Wednesday 11:18:43. Got an error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'. I think the if condition below needs to be changed from if len(vuln_matching_fixed_packages) > 0: to if len(later_non_vuln_sibs) > 0:. Yes that fixes the error! + # TODO: With the fix above, we also need to consolidate this if with the if below -- both start with "if len(later_non_vuln_sibs) > 0:" - if dict_vuln["test_vulnerability"] == vuln: + # if dict_vuln["test_vulnerability"] == vuln: + if dict_vuln["vulnerability"] == vuln: if len(vuln_matching_fixed_packages) > 0: - dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( + # dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( + # closest_fixed_package.purl + # ) + + # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ + # fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns + # ] + + dict_vuln["fixed_by_purl"] = PackageURL.from_string( closest_fixed_package.purl ) - dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ + dict_vuln["fixed_by_purl_vulnerabilities"] = [ fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] else: - dict_vuln["test_fixed_by_purl"] = None - dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] + # dict_vuln["test_fixed_by_purl"] = None + # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] - if len(vuln_matching_fixed_packages) > 0: + dict_vuln["fixed_by_purl"] = None + dict_vuln["fixed_by_purl_vulnerabilities"] = [] - purl_dict["TEST_test_closest_non_vulnerable"] = PackageURL.from_string( + # FIXME: 2023-08-30 Wednesday 11:29:34. Will this handle the error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'? Yes! + # TODO: Having fixed this, we also need to conslidate this if with the similar if above. + # if len(vuln_matching_fixed_packages) > 0: + if len(later_non_vuln_sibs) > 0: + + # purl_dict["TEST_test_closest_non_vulnerable"] = PackageURL.from_string( + # closest_non_vulnerable_sib.purl + # ) + # purl_dict["TEST_test_latest_non_vulnerable"] = PackageURL.from_string( + # latest_non_vulnerable_sib.purl + # ) + + purl_dict["closest_non_vulnerable"] = PackageURL.from_string( closest_non_vulnerable_sib.purl ) - purl_dict["TEST_test_latest_non_vulnerable"] = PackageURL.from_string( + purl_dict["latest_non_vulnerable"] = PackageURL.from_string( latest_non_vulnerable_sib.purl ) else: - purl_dict["TEST_test_closest_non_vulnerable"] = None - purl_dict["TEST_test_latest_non_vulnerable"] = None + # purl_dict["TEST_test_closest_non_vulnerable"] = None + # purl_dict["TEST_test_latest_non_vulnerable"] = None + + purl_dict["closest_non_vulnerable"] = None + purl_dict["latest_non_vulnerable"] = None # Temporary print output during dev/testing. from pprint import pprint diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index d24a7ab40..f36775d6f 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -33,7 +33,7 @@ @@ -50,33 +50,33 @@ Non-vulnerable version - + - +
    - {# if package.purl in fixed_package_details.purl #} - {# if package.purl == fixed_package_details.test_purl.purl #} {% if package.purl == fixed_package_details.test_purl.to_string %} - package.purl = {{ package.purl }} -
    - fixed_package_details.purl = {{ fixed_package_details.purl }} -
    - yikes -
    - fixed_package_details.test_purl = {{ fixed_package_details.test_purl }} -
    - fixed_package_details.test_purl.to_string = {{ fixed_package_details.test_purl.to_string }} -
    - {% for key, value in fixed_package_details.items %} - {# if key == "vulnerabilities" #} {% if key == "test_vulnerabilities" %} - goodby {% for vuln in value %} -
    - vulnerability = {{ vulnerability }} -
    - vuln = {{ vuln }} -
    - vuln.test_vulnerability = {{ vuln.test_vulnerability }} -
    - vuln.test_vulnerability.to_string = {{ vuln.test_vulnerability.to_string }} -
    - vuln.test_vulnerability.vulnerability_id = {{ vuln.test_vulnerability.vulnerability_id }} -
    - vulnerability.vulnerability_id = {{ vulnerability.vulnerability_id }} - - {# if vuln.vulnerability == vulnerability.vulnerability_id #} {% if vuln.test_vulnerability.vulnerability_id == vulnerability.vulnerability_id %} -
    - hello - - - - - - - - - {% if vuln.test_fixed_by_purl is None %} - - - -
    - There are no reported fixed versions. - -
    - {% else %} - {% if vuln.test_fixed_by_purl_vulnerabilities|length == 0 %} - - -
    - {{ vuln.test_fixed_by_purl.version }} -
    Affected by 0 other vulnerabilities. - -
    - {% else %} - - - -
    - {{ vuln.test_fixed_by_purl.version }} - {% if vuln.test_fixed_by_purl_vulnerabilities|length != 1 %} -
    - Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerabilities. - {% else %} -
    - Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerability. +
    + Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + {% else %} +
    + Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerability. {% endif %}
    @@ -301,9 +152,9 @@ This version is affected by these other vulnerabilities:
    {% for fixed_by_vuln in vuln.test_fixed_by_purl_vulnerabilities %} - + {% endfor %}
    @@ -311,17 +162,12 @@ - -
    - {% endif %} {% endif %} - {% endif %} {% endfor %} {% endif %} {% endfor %} - {% endif %}
    - {{ fixed_package_details.test_purl.to_string }} + {{ fixed_package_details.purl.to_string }}
    - {{ fixed_package_details.TEST_test_closest_non_vulnerable.version }} + {{ fixed_package_details.closest_non_vulnerable.version }}
    Latest non-vulnerable version - {{ fixed_package_details.TEST_test_latest_non_vulnerable.version }} + {{ fixed_package_details.latest_non_vulnerable.version }}
    @@ -119,26 +119,26 @@ {{ vulnerability.summary }} - {% if package.purl == fixed_package_details.test_purl.to_string %} + {% if package.purl == fixed_package_details.purl.to_string %} {% for key, value in fixed_package_details.items %} - {% if key == "test_vulnerabilities" %} + {% if key == "vulnerabilities" %} {% for vuln in value %} - {% if vuln.test_vulnerability.vulnerability_id == vulnerability.vulnerability_id %} - {% if vuln.test_fixed_by_purl is None %} + {% if vuln.vulnerability.vulnerability_id == vulnerability.vulnerability_id %} + {% if vuln.fixed_by_purl is None %} There are no reported fixed versions. {% else %} - {% if vuln.test_fixed_by_purl_vulnerabilities|length == 0 %} - {{ vuln.test_fixed_by_purl.version }} + {% if vuln.fixed_by_purl_vulnerabilities|length == 0 %} + {{ vuln.fixed_by_purl.version }}
    - Affected by 0 other vulnerabilities. + Affected by 0 other vulnerabilities. {% else %} - {{ vuln.test_fixed_by_purl.version }} - {% if vuln.test_fixed_by_purl_vulnerabilities|length != 1 %} + {{ vuln.fixed_by_purl.version }} + {% if vuln.fixed_by_purl_vulnerabilities|length != 1 %}
    - Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. {% else %}
    - Affected by {{ vuln.test_fixed_by_purl_vulnerabilities|length }} other vulnerability. + Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerability. {% endif %}
    @@ -151,7 +151,7 @@
    This version is affected by these other vulnerabilities:
    - {% for fixed_by_vuln in vuln.test_fixed_by_purl_vulnerabilities %} + {% for fixed_by_vuln in vuln.fixed_by_purl_vulnerabilities %} diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 9605452e4..67adbd918 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -197,6 +197,87 @@ def setUp(self): fix=False, ) + # Create additional Package objects with various versions to test the major version identification and comparison process. + + self.pypi_setuptools_affected_package = models.Package.objects.create( + type="pypi", + namespace="", + name="setuptools", + version="40.8.0", + qualifiers={}, + subpath="", + ) + + self.pypi_setuptools_fixed_closest_and_latest_non_vulnerable_packages = ( + models.Package.objects.create( + type="pypi", + namespace="", + name="setuptools", + version="65.5.1", + qualifiers={}, + subpath="", + ) + ) + + # pkg:maven/org.eclipse.jetty/jetty-util@9.3.20.v20170531 + + # from the string + jetty_util_purl = PackageURL.from_string( + "pkg:maven/org.eclipse.jetty/jetty-util@9.3.20.v20170531" + ) + + # # convert purl to dict + # jetty_util_purl_to_dict = jetty_util_purl.to_dict() + # # This will avoid the IntegrityError: + # if jetty_util_purl_to_dict.get("qualifiers") is None: + # jetty_util_purl_to_dict["qualifiers"] = {} + # if jetty_util_purl_to_dict.get("subpath") is None: + # jetty_util_purl_to_dict["subpath"] = "" + + # This takes the place of the 2 preceding bits -- uses purl_to_dict() rather than just to_dict() + jetty_util_purl_to_dict = models.purl_to_dict(jetty_util_purl) + + # convert dict to Package + # This needs self, right? + self.maven_jetty_util_affected_package = models.Package.objects.create( + type=jetty_util_purl_to_dict.get("type"), + namespace=jetty_util_purl_to_dict.get("namespace"), + name=jetty_util_purl_to_dict.get("name"), + version=jetty_util_purl_to_dict.get("version"), + qualifiers=jetty_util_purl_to_dict.get("qualifiers"), + subpath=jetty_util_purl_to_dict.get("subpath"), + ) + + # using the create method + # self.maven_jetty_util_affected_package = models.Package.objects.create( + # type="maven", + # namespace="org.eclipse.jetty", + # name="jetty-util", + # version="9.3.20.v20170531", + # qualifiers={}, + # subpath="", + # ) + # pkg:maven/org.eclipse.jetty/jetty-util@9.4.39.v20210325 + self.maven_jetty_util_fixed_package = models.Package.objects.create( + type="maven", + namespace="org.eclipse.jetty", + name="jetty-util", + version="9.4.39.v20210325", + qualifiers={}, + subpath="", + ) + # pkg:maven/org.eclipse.jetty/jetty-util@11.0.14 + self.maven_jetty_util_closest_and_latest_non_vulnerable_packages = ( + models.Package.objects.create( + type="maven", + namespace="org.eclipse.jetty", + name="jetty-util", + version="11.0.14", + qualifiers={}, + subpath="", + ) + ) + def test_get_vulnerable_packages(self): vuln_packages = Package.objects.vulnerable() assert vuln_packages.count() == 3 @@ -405,3 +486,32 @@ def test_string_to_purl_to_dict_to_package(self): ) assert vulnerablecode_package.qualifiers == {} assert vulnerablecode_package.subpath == "" + + def test_compare_package_major_versions(self): + # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., + # a . + + # Convert a PURL string to a PURL. + purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + purl = PackageURL.from_string(purl_string) + + assert type(purl) == PackageURL + assert purl.type == "maven" + assert purl.qualifiers == {} + assert purl.subpath == None + + print("\npurl_string = {}".format(purl_string)) + + print("\npurl = {}".format(purl)) + + print("\nHello VulnerableCode!\n") + + all_packages = Package.objects + print("\nPackage.objects = {}\n".format(Package.objects)) + print("\nall_packages.distinct() = {}\n".format(all_packages.distinct())) + print("\nall_packages.distinct()[0] = {}\n".format(all_packages.distinct()[0])) + + for pkg in all_packages.distinct(): + print(PackageURL.from_string(pkg.purl)) + + print("") From 1ffccbed2bea756e0202302ef14262e0b46f0cb0 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 5 Sep 2023 19:03:35 -0700 Subject: [PATCH 32/53] Complete first round of major-version vetting #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 295 ++++++++++-------- .../templates/package_details.html | 23 +- 2 files changed, 186 insertions(+), 132 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 716fdc33a..ed05e5d0a 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -700,18 +700,73 @@ def fixed_package_details(self): if sib.is_vulnerable is False: non_vuln_sibs.append(sib) - # Add the greater-than versions to a new list. - univers_version = RANGE_CLASS_BY_SCHEMES[self.type].version_class - print("\nunivers_version = {}".format(univers_version)) + # Get the univers version_class for the purl, i.e., the Package the user has searched for. + univers_version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class + # print("\nunivers_version_class = {}".format(univers_version_class)) + + # ======================================================= + # Get the univers version for the purl. + purl_univers_version = univers_version_class(PackageURL.from_string(self.purl).version) + # print("\n>>> purl_univers_version = {}\n".format(purl_univers_version)) + # 2023-09-04 Monday 18:23:51. Is the version just a string I can split? + # print("\n>>> type(purl_univers_version) = {}\n".format(type(purl_univers_version))) + # print( + # "\n>>> type(purl_univers_version.__str__()) = {}\n".format( + # type(purl_univers_version.__str__()) + # ) + # ) + # print( + # "\n>>> purl_univers_version.__str__().split('.')[0] = {}\n".format( + # purl_univers_version.__str__().split(".")[0] + # ) + # ) + + # Convert the univers version for the purl to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. + purl_univers_version_major_int = int(purl_univers_version.__str__().split(".")[0]) + # print("\n>>> purl_univers_version_major_int = {}\n".format(purl_univers_version_major_int)) + # print( + # "\n>>> type(purl_univers_version_major_int) = {}\n".format( + # type(purl_univers_version_major_int) + # ) + # ) + + # Create a list for the later non-vuln siblings. later_non_vuln_sibs = [] + # Each non_vuln_sib is a Package. for non_vuln_sib in non_vuln_sibs: - if univers_version(non_vuln_sib.version) > univers_version(self.version): + + # print("\n### type(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) + + # Get the univers version for the non_vuln_sib. + non_vuln_sib_univers_version = univers_version_class( + PackageURL.from_string(non_vuln_sib.purl).version + ) + + # print("\n### non_vuln_sib_univers_version = {}\n".format(non_vuln_sib_univers_version)) + + # Convert the univers version for the purl to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. + non_vuln_sib_univers_version_major_int = int( + non_vuln_sib_univers_version.__str__().split(".")[0] + ) + + # print( + # "\n### non_vuln_sib_univers_version_major_int = {}\n".format( + # non_vuln_sib_univers_version_major_int + # ) + # ) + + # Add it to the 'later_non_vuln_sibs' list if the major versions are the same. + # if univers_version_class(non_vuln_sib.version) > univers_version_class(self.version): + if ( + univers_version_class(non_vuln_sib.version) > univers_version_class(self.version) + and purl_univers_version_major_int == non_vuln_sib_univers_version_major_int + ): later_non_vuln_sibs.append(non_vuln_sib) # Take the list of vulnerabilities affecting the current package, retrieve a list of the fixed - # packages for each, and assign the result to a custom attribute, 'matching_fixed_packages'. - # Ex: qs[0].matching_fixed_packages returns the fixed package(s) for the 1st vulnerability - # for this affected package (i.e., self). + # packages for each (self.get_fixed_packages()), and assign the result to a custom attribute, 'matching_fixed_packages'. + # Ex: qs[0].matching_fixed_packages returns the fixed package(s) for the 1st vulnerability. + qs = qs.prefetch_related( Prefetch( "packages", @@ -722,43 +777,101 @@ def fixed_package_details(self): purl_dict = {} # Pass a PackageURL object to the Jinja2 template via the context dictionary. - # purl_dict["test_purl"] = PackageURL.from_string(self.purl) - # purl_dict.update({"test_vulnerabilities": []}) purl_dict["purl"] = PackageURL.from_string(self.purl) - print("\npurl = {}".format(purl_dict["purl"])) + # print("\npurl = {}".format(purl_dict["purl"])) - purl_univers_version = univers_version(purl_dict["purl"].version) - print("\npurl_univers_version = {}".format(purl_univers_version)) + closest_non_vulnerable_sib = "" + latest_non_vulnerable_sib = "" - # ALERT: This throws an error: AttributeError: 'MavenVersion' object has no attribute 'major' - # How do we compare major versions? - # purl_univers_version_major = purl_univers_version.major + # closest_non_vulnerable_sib_non_breaking = "" + # latest_non_vulnerable_sib_non_breaking = "" - # purl_semver_version = versions.SemverVersion(purl_dict["purl"].version) - # print("\npurl_semver_version = {}".format(purl_semver_version)) + # We've already filled later_non_vuln_sibs with all non_vuln sibs with the same major + # version as the purl the user is searching for (i.e., self) -- but this could be an empty list. + if len(later_non_vuln_sibs) > 0: + closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] + latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] - # purl_semver_version_major = purl_semver_version.major - # print("\npurl_semver_version_major = {}".format(purl_semver_version_major)) + purl_dict["closest_non_vulnerable"] = PackageURL.from_string( + closest_non_vulnerable_sib.purl + ) + purl_dict["latest_non_vulnerable"] = PackageURL.from_string( + latest_non_vulnerable_sib.purl + ) - purl_semver_version_major = versions.SemverVersion(purl_dict["purl"].version).major - print("\npurl_semver_version_major = {}\n".format(purl_semver_version_major)) + # print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) + + # later_non_vuln_sibs_non_breaking = [] + # for non_vuln_sib in later_non_vuln_sibs: + # print("\nnon_vuln_sib = {}\n".format(non_vuln_sib)) + # print("\ntype(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) + + else: + # closest_non_vulnerable_sib = "" + # latest_non_vulnerable_sib = "" + + purl_dict["closest_non_vulnerable"] = None + purl_dict["latest_non_vulnerable"] = None + # ======================================================== + # if len(later_non_vuln_sibs) > 0: + # purl_dict["closest_non_vulnerable"] = PackageURL.from_string( + # closest_non_vulnerable_sib.purl + # ) + # purl_dict["latest_non_vulnerable"] = PackageURL.from_string( + # latest_non_vulnerable_sib.purl + # ) + # else: + # purl_dict["closest_non_vulnerable"] = None + # purl_dict["latest_non_vulnerable"] = None + # ======================================================== + + # No longer need this. + # purl_univers_version_test = univers_version_class(purl_dict["purl"].version) + # print("\npurl_univers_version_test = {}".format(purl_univers_version_test)) purl_dict.update({"vulnerabilities": []}) + # For each vulnerability affecting the current Package... for vuln in qs: later_matching_fixed_packages = [] # Pass a Vulnerability object. - # purl_dict["test_vulnerabilities"].append({"test_vulnerability": vuln}) purl_dict["vulnerabilities"].append({"vulnerability": vuln}) + # Using the qs prefetch, get a of all the matching fixed packages for this vulnerability, all major versions to start with. vuln_matching_fixed_packages = vuln.matching_fixed_packages + # print( + # "\n??? type(vuln_matching_fixed_packages) = {}\n".format( + # type(vuln_matching_fixed_packages) + # ) + # ) closest_fixed_package = "" + # ALERT: 2023-09-05 Tuesday 17:42:58. When refreshing a Package details page, I got an error: local variable 'sort_fixed_by_packages_by_version' referenced before assignment (referring to the definition of closest_fixed_package below) + # ALERT: Do I need to define it here? + sort_fixed_by_packages_by_version = [] + if len(vuln_matching_fixed_packages) > 0: for fixed_pkg in vuln_matching_fixed_packages: - if fixed_pkg in matching_fixed_packages and univers_version( - fixed_pkg.version - ) > univers_version(self.version): + # fixed_pkg is a 'vulnerabilities.models.Package' + # print("\n!!! type(fixed_pkg) = {}\n".format(type(fixed_pkg))) + + # Get the univers version for the fixed_pkg. + fixed_pkg_univers_version = univers_version_class( + PackageURL.from_string(fixed_pkg.purl).version + ) + + # Convert the univers version for the fixed_pkg to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. + fixed_pkg_univers_version_major_int = int( + fixed_pkg_univers_version.__str__().split(".")[0] + ) + + # Add it to the 'later_non_vuln_sibs' list if the major versions are the same. + if ( + fixed_pkg in matching_fixed_packages + and univers_version_class(fixed_pkg.version) + > univers_version_class(self.version) + and purl_univers_version_major_int == fixed_pkg_univers_version_major_int + ): later_matching_fixed_packages.append(fixed_pkg) # We already identified a closest fixed package and are looking for a later fixed package. @@ -766,126 +879,56 @@ def fixed_package_details(self): sort_fixed_by_packages_by_version = self.sort_by_version( later_matching_fixed_packages ) - print( - "\nsort_fixed_by_packages_by_version = {}\n".format( - sort_fixed_by_packages_by_version - ) - ) - for fixed_by_pkg in sort_fixed_by_packages_by_version: - print("\nfixed_by_pkg.version = {}\n".format(fixed_by_pkg.version)) - purl_from_fixed_by_pkg = PackageURL.from_string(fixed_by_pkg.purl) - print("\npurl_from_fixed_by_pkg = {}\n".format(purl_from_fixed_by_pkg)) - - print("\nfixed_by_pkg = {}\n".format(fixed_by_pkg)) - - # purl_semver_version_major = versions.SemverVersion(purl_dict["purl"].version).major - # print("\npurl_semver_version_major = {}\n".format(purl_semver_version_major)) - - fixed_by_pkg_semver_version_major = versions.SemverVersion( - fixed_by_pkg.version - ).major - print( - "\nfixed_by_pkg_semver_version_major = {}\n".format( - fixed_by_pkg_semver_version_major - ) - ) + # print( + # "\nsort_fixed_by_packages_by_version = {}\n".format( + # sort_fixed_by_packages_by_version + # ) + # ) - closest_fixed_package = sort_fixed_by_packages_by_version[0] + # ALERT: 2023-09-05 Tuesday 17:53:54. I think these 2 lines need to be indented to be inside the if condition above. + closest_fixed_package = sort_fixed_by_packages_by_version[0] + closest_fixed_package_vulns = closest_fixed_package.affected_by - closest_fixed_package_vulns = closest_fixed_package.affected_by + # closest_fixed_package = sort_fixed_by_packages_by_version[0] + # closest_fixed_package_vulns = closest_fixed_package.affected_by else: - closest_fixed_package = "There are no reported fixed packages." + # ALERT: 2023-09-05 Tuesday 18:03:46. I don't think we want this string but rather something like None, for the if condition below. + # closest_fixed_package = "There are no reported fixed packages." + closest_fixed_package = None - # for dict_vuln in purl_dict["test_vulnerabilities"]: for dict_vuln in purl_dict["vulnerabilities"]: - closest_non_vulnerable_sib = "" - # if len(later_non_vuln_sibs) > 0: - # closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - # else: - # closest_non_vulnerable_sib = "" - - latest_non_vulnerable_sib = "" - # if len(later_non_vuln_sibs) > 0: - # latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] - # else: - # latest_non_vulnerable_sib = "" - - closest_non_vulnerable_sib_non_breaking = "" - latest_non_vulnerable_sib_non_breaking = "" - - if len(later_non_vuln_sibs) > 0: - closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] - - print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) - - later_non_vuln_sibs_non_breaking = [] - for non_vuln_sib in later_non_vuln_sibs: - print("\nnon_vuln_sib = {}\n".format(non_vuln_sib)) - print("\ntype(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) - - # 2023-08-28 Monday 19:19:03. I need to resume here and populate the list later_non_vuln_sibs_non_breaking, then see whether there are any non-breaking versions and in anmy event add 2 more dict entries for the closest non-breaking and latest non-breaking versions. - else: - closest_non_vulnerable_sib = "" - latest_non_vulnerable_sib = "" - # FIXME: 2023-08-30 Wednesday 11:18:43. Got an error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'. I think the if condition below needs to be changed from if len(vuln_matching_fixed_packages) > 0: to if len(later_non_vuln_sibs) > 0:. Yes that fixes the error! - # TODO: With the fix above, we also need to consolidate this if with the if below -- both start with "if len(later_non_vuln_sibs) > 0:" - - # if dict_vuln["test_vulnerability"] == vuln: if dict_vuln["vulnerability"] == vuln: - if len(vuln_matching_fixed_packages) > 0: - - # dict_vuln["test_fixed_by_purl"] = PackageURL.from_string( - # closest_fixed_package.purl - # ) - - # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [ - # fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns - # ] - + # ALERT: 2023-09-05 Tuesday 17:58:24. This is not the correct test because vuln_matching_fixed_packages might contain no major version identical to the purl major version! Instead we want to check whether the closest_fixed_package exists, which we defined above as closest_fixed_package = sort_fixed_by_packages_by_version[0] + # if len(vuln_matching_fixed_packages) > 0: + if closest_fixed_package: dict_vuln["fixed_by_purl"] = PackageURL.from_string( closest_fixed_package.purl ) + # ALERT: 2023-09-05 Tuesday 18:00:09. I expect we'll also see an error here and will need to define closest_fixed_package_vulns higher above than where it is currently defined. dict_vuln["fixed_by_purl_vulnerabilities"] = [ fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] else: - - # dict_vuln["test_fixed_by_purl"] = None - # dict_vuln["test_fixed_by_purl_vulnerabilities"] = [] - dict_vuln["fixed_by_purl"] = None dict_vuln["fixed_by_purl_vulnerabilities"] = [] - # FIXME: 2023-08-30 Wednesday 11:29:34. Will this handle the error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'? Yes! - # TODO: Having fixed this, we also need to conslidate this if with the similar if above. - # if len(vuln_matching_fixed_packages) > 0: - if len(later_non_vuln_sibs) > 0: - - # purl_dict["TEST_test_closest_non_vulnerable"] = PackageURL.from_string( - # closest_non_vulnerable_sib.purl - # ) - # purl_dict["TEST_test_latest_non_vulnerable"] = PackageURL.from_string( - # latest_non_vulnerable_sib.purl - # ) - - purl_dict["closest_non_vulnerable"] = PackageURL.from_string( - closest_non_vulnerable_sib.purl - ) - purl_dict["latest_non_vulnerable"] = PackageURL.from_string( - latest_non_vulnerable_sib.purl - ) - - else: - - # purl_dict["TEST_test_closest_non_vulnerable"] = None - # purl_dict["TEST_test_latest_non_vulnerable"] = None - - purl_dict["closest_non_vulnerable"] = None - purl_dict["latest_non_vulnerable"] = None + # 2023-09-05 Tuesday 15:26:59. Move this chunk up to just under the start of the dict. + # # 2023-08-30 Wednesday 11:29:34. Will this handle the error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'? Yes! + # # Having fixed this, we also need to consolidate this if with the similar if above. + + # if len(later_non_vuln_sibs) > 0: + # purl_dict["closest_non_vulnerable"] = PackageURL.from_string( + # closest_non_vulnerable_sib.purl + # ) + # purl_dict["latest_non_vulnerable"] = PackageURL.from_string( + # latest_non_vulnerable_sib.purl + # ) + # else: + # purl_dict["closest_non_vulnerable"] = None + # purl_dict["latest_non_vulnerable"] = None # Temporary print output during dev/testing. from pprint import pprint diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index f36775d6f..47a672b3d 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -50,7 +50,11 @@ Non-vulnerable version - {{ fixed_package_details.closest_non_vulnerable.version }} + {% if fixed_package_details.closest_non_vulnerable.version %} + {{ fixed_package_details.closest_non_vulnerable.version }} + {% else %} + None. + {% endif %} + + {% if fixed_package_details.latest_non_vulnerable.version %} + {{ fixed_package_details.latest_non_vulnerable.version }} + {% else %} + None. + {% endif %} + This package is not known to be affected by vulnerabilities. {% endfor %} From dc701ca81fcda1f90e606d825855c1ad23784cb4 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Thu, 7 Sep 2023 17:58:23 -0700 Subject: [PATCH 33/53] Remove major-version code, clean comments etc. #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 245 ++++-------------- .../templates/package_details.html | 21 +- 2 files changed, 53 insertions(+), 213 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index ed05e5d0a..5c71a6bd8 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -645,36 +645,31 @@ def get_absolute_url(self): """ return reverse("package_details", args=[self.purl]) - def get_fixed_packages(self, package): - """ - Return a queryset of all packages that fix a vulnerability with - same type, namespace, name, subpath and qualifiers of the `package` - """ - return Package.objects.filter( - name=package.name, - namespace=package.namespace, - type=package.type, - qualifiers=package.qualifiers, - subpath=package.subpath, - packagerelatedvulnerability__fix=True, - ).distinct() - - def get_sibling_packages(self, package): - """ - Return a queryset of all packages with the same type, namespace, name, subpath and - qualifiers as the target package, whether or not the 'sibling packages' fix any vulnerability. - Identical to get_fixed_packages() above except for the removal of 'packagerelatedvulnerability__fix=True,'. - """ - return Package.objects.filter( - name=package.name, - namespace=package.namespace, - type=package.type, - qualifiers=package.qualifiers, - subpath=package.subpath, - ).distinct() + def get_fixed_packages(self, package, fix=True): + """ + With fix=True, return a queryset of all packages that fix a vulnerability and have the + same type, namespace, name, subpath and qualifiers as the `package`. + With fix=False, return a queryset of all packages that have the same type, namespace, + name, subpath and qualifiers as the `package`, whether or not they fix any vulnerability. + """ + filter_dict = { + "name": package.name, + "namespace": package.namespace, + "type": package.type, + "qualifiers": package.qualifiers, + "subpath": package.subpath, + } + + if fix: + filter_dict["packagerelatedvulnerability__fix"] = True + + return Package.objects.filter(**filter_dict).distinct() def sort_by_version(self, packages_to_sort): - # Incoming is a list of objects. + """ + Use the univers version_class to sort the incoming list of vulnerabilities.models.Package + objects. + """ univers_version = RANGE_CLASS_BY_SCHEMES[packages_to_sort[0].type].version_class return sorted( packages_to_sort, @@ -684,110 +679,44 @@ def sort_by_version(self, packages_to_sort): @property def fixed_package_details(self): """ - Find and report all fixed by packages (and certain related versions) for each vulnerability in - each affected package. Report packages with no vulnerabilities as well. + For a given package, report the closest subsequent fixed by version for each of that + package's vulnerabilities as well as the closest non-vulnerable version and the most recent + non-vulnerable version. """ - # Get all fixed packages that match the target package (type etc.), regardless of fixed vuln. - matching_fixed_packages = self.get_fixed_packages(package=self) - - # Get a list of the vulnerabilities that affect the current package (i.e., self). + fixed_packages = self.get_fixed_packages(package=self, fix=True) qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - - all_sibling_packages = self.get_sibling_packages(package=self) + all_sibling_packages = self.get_fixed_packages(package=self, fix=False) non_vuln_sibs = [] for sib in all_sibling_packages: if sib.is_vulnerable is False: non_vuln_sibs.append(sib) - # Get the univers version_class for the purl, i.e., the Package the user has searched for. univers_version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class - # print("\nunivers_version_class = {}".format(univers_version_class)) - - # ======================================================= - # Get the univers version for the purl. - purl_univers_version = univers_version_class(PackageURL.from_string(self.purl).version) - # print("\n>>> purl_univers_version = {}\n".format(purl_univers_version)) - # 2023-09-04 Monday 18:23:51. Is the version just a string I can split? - # print("\n>>> type(purl_univers_version) = {}\n".format(type(purl_univers_version))) - # print( - # "\n>>> type(purl_univers_version.__str__()) = {}\n".format( - # type(purl_univers_version.__str__()) - # ) - # ) - # print( - # "\n>>> purl_univers_version.__str__().split('.')[0] = {}\n".format( - # purl_univers_version.__str__().split(".")[0] - # ) - # ) - - # Convert the univers version for the purl to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. - purl_univers_version_major_int = int(purl_univers_version.__str__().split(".")[0]) - # print("\n>>> purl_univers_version_major_int = {}\n".format(purl_univers_version_major_int)) - # print( - # "\n>>> type(purl_univers_version_major_int) = {}\n".format( - # type(purl_univers_version_major_int) - # ) - # ) - - # Create a list for the later non-vuln siblings. later_non_vuln_sibs = [] - # Each non_vuln_sib is a Package. - for non_vuln_sib in non_vuln_sibs: - - # print("\n### type(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) - # Get the univers version for the non_vuln_sib. + for non_vuln_sib in non_vuln_sibs: non_vuln_sib_univers_version = univers_version_class( PackageURL.from_string(non_vuln_sib.purl).version ) - # print("\n### non_vuln_sib_univers_version = {}\n".format(non_vuln_sib_univers_version)) - - # Convert the univers version for the purl to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. - non_vuln_sib_univers_version_major_int = int( - non_vuln_sib_univers_version.__str__().split(".")[0] - ) - - # print( - # "\n### non_vuln_sib_univers_version_major_int = {}\n".format( - # non_vuln_sib_univers_version_major_int - # ) - # ) - - # Add it to the 'later_non_vuln_sibs' list if the major versions are the same. - # if univers_version_class(non_vuln_sib.version) > univers_version_class(self.version): - if ( - univers_version_class(non_vuln_sib.version) > univers_version_class(self.version) - and purl_univers_version_major_int == non_vuln_sib_univers_version_major_int - ): + if univers_version_class(non_vuln_sib.version) > univers_version_class(self.version): later_non_vuln_sibs.append(non_vuln_sib) - # Take the list of vulnerabilities affecting the current package, retrieve a list of the fixed - # packages for each (self.get_fixed_packages()), and assign the result to a custom attribute, 'matching_fixed_packages'. - # Ex: qs[0].matching_fixed_packages returns the fixed package(s) for the 1st vulnerability. - qs = qs.prefetch_related( Prefetch( "packages", - queryset=matching_fixed_packages, - to_attr="matching_fixed_packages", + queryset=fixed_packages, + to_attr="fixed_packages", ) ) purl_dict = {} - # Pass a PackageURL object to the Jinja2 template via the context dictionary. purl_dict["purl"] = PackageURL.from_string(self.purl) - # print("\npurl = {}".format(purl_dict["purl"])) closest_non_vulnerable_sib = "" latest_non_vulnerable_sib = "" - # closest_non_vulnerable_sib_non_breaking = "" - # latest_non_vulnerable_sib_non_breaking = "" - - # We've already filled later_non_vuln_sibs with all non_vuln sibs with the same major - # version as the purl the user is searching for (i.e., self) -- but this could be an empty list. if len(later_non_vuln_sibs) > 0: closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] @@ -799,115 +728,60 @@ def fixed_package_details(self): latest_non_vulnerable_sib.purl ) - # print("\nlater_non_vuln_sibs = {}\n".format(later_non_vuln_sibs)) - - # later_non_vuln_sibs_non_breaking = [] - # for non_vuln_sib in later_non_vuln_sibs: - # print("\nnon_vuln_sib = {}\n".format(non_vuln_sib)) - # print("\ntype(non_vuln_sib) = {}\n".format(type(non_vuln_sib))) - else: - # closest_non_vulnerable_sib = "" - # latest_non_vulnerable_sib = "" purl_dict["closest_non_vulnerable"] = None purl_dict["latest_non_vulnerable"] = None - # ======================================================== - # if len(later_non_vuln_sibs) > 0: - # purl_dict["closest_non_vulnerable"] = PackageURL.from_string( - # closest_non_vulnerable_sib.purl - # ) - # purl_dict["latest_non_vulnerable"] = PackageURL.from_string( - # latest_non_vulnerable_sib.purl - # ) - # else: - # purl_dict["closest_non_vulnerable"] = None - # purl_dict["latest_non_vulnerable"] = None - # ======================================================== - - # No longer need this. - # purl_univers_version_test = univers_version_class(purl_dict["purl"].version) - # print("\npurl_univers_version_test = {}".format(purl_univers_version_test)) purl_dict.update({"vulnerabilities": []}) - # For each vulnerability affecting the current Package... for vuln in qs: - later_matching_fixed_packages = [] - # Pass a Vulnerability object. purl_dict["vulnerabilities"].append({"vulnerability": vuln}) - # Using the qs prefetch, get a of all the matching fixed packages for this vulnerability, all major versions to start with. - vuln_matching_fixed_packages = vuln.matching_fixed_packages - # print( - # "\n??? type(vuln_matching_fixed_packages) = {}\n".format( - # type(vuln_matching_fixed_packages) - # ) - # ) closest_fixed_package = "" - - # ALERT: 2023-09-05 Tuesday 17:42:58. When refreshing a Package details page, I got an error: local variable 'sort_fixed_by_packages_by_version' referenced before assignment (referring to the definition of closest_fixed_package below) - # ALERT: Do I need to define it here? + closest_fixed_package_vulns = "" + later_fixed_packages = [] sort_fixed_by_packages_by_version = [] + vuln_fixed_packages = vuln.fixed_packages - if len(vuln_matching_fixed_packages) > 0: - for fixed_pkg in vuln_matching_fixed_packages: - # fixed_pkg is a 'vulnerabilities.models.Package' - # print("\n!!! type(fixed_pkg) = {}\n".format(type(fixed_pkg))) + if len(vuln_fixed_packages) > 0: + for fixed_pkg in vuln_fixed_packages: - # Get the univers version for the fixed_pkg. fixed_pkg_univers_version = univers_version_class( PackageURL.from_string(fixed_pkg.purl).version ) - # Convert the univers version for the fixed_pkg to a string, split the string at each '.', grab the first of these, and convert it to an int -- this is the major version. - fixed_pkg_univers_version_major_int = int( - fixed_pkg_univers_version.__str__().split(".")[0] - ) + if fixed_pkg in fixed_packages and univers_version_class( + fixed_pkg.version + ) > univers_version_class(self.version): - # Add it to the 'later_non_vuln_sibs' list if the major versions are the same. - if ( - fixed_pkg in matching_fixed_packages - and univers_version_class(fixed_pkg.version) - > univers_version_class(self.version) - and purl_univers_version_major_int == fixed_pkg_univers_version_major_int - ): - later_matching_fixed_packages.append(fixed_pkg) - - # We already identified a closest fixed package and are looking for a later fixed package. - if later_matching_fixed_packages: - sort_fixed_by_packages_by_version = self.sort_by_version( - later_matching_fixed_packages - ) - # print( - # "\nsort_fixed_by_packages_by_version = {}\n".format( - # sort_fixed_by_packages_by_version - # ) - # ) + later_fixed_packages.append(fixed_pkg) - # ALERT: 2023-09-05 Tuesday 17:53:54. I think these 2 lines need to be indented to be inside the if condition above. + if later_fixed_packages: + sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) closest_fixed_package = sort_fixed_by_packages_by_version[0] closest_fixed_package_vulns = closest_fixed_package.affected_by - - # closest_fixed_package = sort_fixed_by_packages_by_version[0] - # closest_fixed_package_vulns = closest_fixed_package.affected_by + print( + "\nclosest_fixed_package_vulns = {}\n".format(closest_fixed_package_vulns) + ) + print( + "\ntype(closest_fixed_package_vulns) = {}\n".format( + type(closest_fixed_package_vulns) + ) + ) else: - # ALERT: 2023-09-05 Tuesday 18:03:46. I don't think we want this string but rather something like None, for the if condition below. - # closest_fixed_package = "There are no reported fixed packages." closest_fixed_package = None + closest_fixed_package_vulns = None for dict_vuln in purl_dict["vulnerabilities"]: if dict_vuln["vulnerability"] == vuln: - # ALERT: 2023-09-05 Tuesday 17:58:24. This is not the correct test because vuln_matching_fixed_packages might contain no major version identical to the purl major version! Instead we want to check whether the closest_fixed_package exists, which we defined above as closest_fixed_package = sort_fixed_by_packages_by_version[0] - # if len(vuln_matching_fixed_packages) > 0: if closest_fixed_package: dict_vuln["fixed_by_purl"] = PackageURL.from_string( closest_fixed_package.purl ) - # ALERT: 2023-09-05 Tuesday 18:00:09. I expect we'll also see an error here and will need to define closest_fixed_package_vulns higher above than where it is currently defined. dict_vuln["fixed_by_purl_vulnerabilities"] = [ fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns ] @@ -915,21 +789,6 @@ def fixed_package_details(self): dict_vuln["fixed_by_purl"] = None dict_vuln["fixed_by_purl_vulnerabilities"] = [] - # 2023-09-05 Tuesday 15:26:59. Move this chunk up to just under the start of the dict. - # # 2023-08-30 Wednesday 11:29:34. Will this handle the error when clicking from the Vulnerability details page Fixed by packages tab -- 'str' object has no attribute 'purl'? Yes! - # # Having fixed this, we also need to consolidate this if with the similar if above. - - # if len(later_non_vuln_sibs) > 0: - # purl_dict["closest_non_vulnerable"] = PackageURL.from_string( - # closest_non_vulnerable_sib.purl - # ) - # purl_dict["latest_non_vulnerable"] = PackageURL.from_string( - # latest_non_vulnerable_sib.purl - # ) - # else: - # purl_dict["closest_non_vulnerable"] = None - # purl_dict["latest_non_vulnerable"] = None - # Temporary print output during dev/testing. from pprint import pprint diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 47a672b3d..9d9c70b77 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -47,7 +47,7 @@ - Non-vulnerable version + Next non-vulnerable version {% if fixed_package_details.closest_non_vulnerable.version %} @@ -57,21 +57,11 @@ {% endif %} - Latest non-vulnerable version - - {% if fixed_package_details.latest_non_vulnerable.version %} {{ fixed_package_details.latest_non_vulnerable.version }} {% else %} @@ -79,14 +69,6 @@ {% endif %} -
    @@ -184,7 +166,6 @@ {% empty %} - This package is not known to be affected by vulnerabilities. From f0bdd3be6278802f76fec0dd49b871d08633756b Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 11 Sep 2023 09:28:03 -0700 Subject: [PATCH 34/53] Begin test refactoring #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/tests/test_models.py | 537 ++++++++++++++++++++------- 1 file changed, 398 insertions(+), 139 deletions(-) diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 67adbd918..bb7eaa0bf 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -22,6 +22,7 @@ from univers.version_range import RANGE_CLASS_BY_SCHEMES from vulnerabilities import models +from vulnerabilities.models import Alias from vulnerabilities.models import Package @@ -113,6 +114,70 @@ def test_cwe_not_present_in_weaknesses_db(self): @pytest.mark.django_db class TestPackageModel(TestCase): def setUp(self): + # ZAP: 2023-09-08 Friday 17:40:36. Let's start over with a focused package/vuln/fix group we know from the DB/UI testing: pkg:pypi/redis@4.1.1 + # It has 2 non-vulns versions, both the same: 5.0.0b1 + # The first of its two vulns is VCID-g2fu-45jw-aaan (aliases: CVE-2023-28858 and GHSA-24wv-mv5m-xv4h), fixed by 4.3.6 w/1 vuln of its own + # The second is VCID-rqe1-dkmg-aaad (aliases: CVE-2023-28859 and GHSA-8fww-64cx-x8p5), fixed by 5.0.0b1 w/ 0 vulns of its own + + # pkg + self.package_pypi_redis_4_1_1 = models.Package.objects.create( + type="pypi", + namespace="", + name="redix", + version="4.1.1", + qualifiers={}, + subpath="", + ) + + # vuln + self.vuln_VCID_g2fu_45jw_aaan = models.Vulnerability.objects.create( + summary="This is VCID-g2fu-45jw-aaan", + vulnerability_id="VCID-g2fu-45jw-aaan", + ) + + # relationship + models.PackageRelatedVulnerability.objects.create( + package=self.package_pypi_redis_4_1_1, + vulnerability=self.vuln_VCID_g2fu_45jw_aaan, + fix=False, + ) + + # aliases + Alias.objects.create(alias="CVE-2023-28858", vulnerability=self.vuln_VCID_g2fu_45jw_aaan) + Alias.objects.create( + alias="GHSA-24wv-mv5m-xv4h", vulnerability=self.vuln_VCID_g2fu_45jw_aaan + ) + + # ZAP: Need fix for above vuln + + # vuln + self.vuln_VCID_rqe1_dkmg_aaad = models.Vulnerability.objects.create( + summary="This is VCID-rqe1-dkmg-aaad", + vulnerability_id="VCID-rqe1-dkmg-aaad", + ) + + # relationship + models.PackageRelatedVulnerability.objects.create( + package=self.package_pypi_redis_4_1_1, + vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, + fix=False, + ) + + # aliases + Alias.objects.create(alias="CVE-2023-28859", vulnerability=self.vuln_VCID_rqe1_dkmg_aaad) + Alias.objects.create( + alias="GHSA-8fww-64cx-x8p5", vulnerability=self.vuln_VCID_rqe1_dkmg_aaad + ) + + # ZAP: Need fix for above vuln + + # ZAP: Need non-vuln version(s) + + # ======================================================== + + # ======================================================== + + # ZAP: Not sure we want to keep any of this. self.vuln1 = models.Vulnerability.objects.create( summary="test-vuln1", vulnerability_id="VCID-123", @@ -278,10 +343,58 @@ def setUp(self): ) ) + # 2023-09-08 Friday 21:24:54. New experiment + def test_explore_packages(self): + print("\nself.package_pypi_redis_4_1_1 = {}\n".format(self.package_pypi_redis_4_1_1)) + + print( + "\nself.package_pypi_redis_4_1_1.package_url = {}\n".format( + self.package_pypi_redis_4_1_1.package_url + ) + ) + + print( + "\nself.package_pypi_redis_4_1_1.plain_package_url = {}\n".format( + self.package_pypi_redis_4_1_1.plain_package_url + ) + ) + + print( + "\nself.package_pypi_redis_4_1_1.purl = {}\n".format(self.package_pypi_redis_4_1_1.purl) + ) + + print( + "\nself.package_pypi_redis_4_1_1.affected_by = {}\n".format( + self.package_pypi_redis_4_1_1.affected_by + ) + ) + + for vuln in self.package_pypi_redis_4_1_1.affected_by: + print(vuln) + print(vuln.summary) + print("") + + print( + "\nself.package_pypi_redis_4_1_1.fixing = {}\n".format( + self.package_pypi_redis_4_1_1.fixing + ) + ) + + print( + "\nself.package_pypi_redis_4_1_1.get_absolute_url() = {}\n".format( + self.package_pypi_redis_4_1_1.get_absolute_url() + ) + ) + + # ZAP: 2023-09-08 Friday 18:37:13. Need to revise this after adding new pkgs, vulns, aliases above. def test_get_vulnerable_packages(self): vuln_packages = Package.objects.vulnerable() - assert vuln_packages.count() == 3 - assert vuln_packages.distinct().count() == 2 + print("\nvuln_packages = {}\n".format(vuln_packages)) + # assert vuln_packages.count() == 3 + # 2023-09-08 Friday 16:59:33. Update given today's additions etc. + assert vuln_packages.count() == 5 + # assert vuln_packages.distinct().count() == 2 + assert vuln_packages.distinct().count() == 3 first_vulnerable_package = vuln_packages.distinct()[0] matching_fixed_packages = first_vulnerable_package.get_fixed_packages( @@ -299,33 +412,177 @@ def test_get_vulnerable_packages(self): == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1" ) - purl_dict = { - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", - "vulnerabilities": [ - { - "vulnerability": "VCID-123", - "closest_fixed_by": { - "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - "type": "maven", - "namespace": "com.fasterxml.jackson.core", - "name": "jackson-databind", - "version": "2.13.2", - "qualifiers": {}, - "subpath": "", - }, - "closest_fixed_by_vulnerabilities": [{"vuln_id": "VCID-000"}], - }, - { - "vulnerability": "VCID-456", - "closest_fixed_by": {}, - "closest_fixed_by_vulnerabilities": [], - }, - ], - "closest_non_vulnerable": {}, - "latest_non_vulnerable": {}, - } - - assert vuln_packages.distinct()[0].fixed_package_details == purl_dict + # purl_dict = { + # "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + # "vulnerabilities": [ + # { + # "vulnerability": "VCID-123", + # "closest_fixed_by": { + # "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + # "type": "maven", + # "namespace": "com.fasterxml.jackson.core", + # "name": "jackson-databind", + # "version": "2.13.2", + # "qualifiers": {}, + # "subpath": "", + # }, + # "closest_fixed_by_vulnerabilities": [{"vuln_id": "VCID-000"}], + # }, + # { + # "vulnerability": "VCID-456", + # "closest_fixed_by": {}, + # "closest_fixed_by_vulnerabilities": [], + # }, + # ], + # "closest_non_vulnerable": {}, + # "latest_non_vulnerable": {}, + # } + + # purl_dict = { + # "purl": PackageURL( + # type="maven", + # namespace="com.fasterxml.jackson.core", + # name="jackson-databind", + # version="2.13.1", + # qualifiers={}, + # subpath=None, + # ), + # "closest_non_vulnerable": PackageURL( + # type="maven", + # namespace="com.fasterxml.jackson.core", + # name="jackson-databind", + # version="2.14.0-rc1", + # qualifiers={}, + # subpath=None, + # ), + # "latest_non_vulnerable": PackageURL( + # type="maven", + # namespace="com.fasterxml.jackson.core", + # name="jackson-databind", + # version="2.14.0-rc1", + # qualifiers={}, + # subpath=None, + # ), + # "vulnerabilities": [ + # { + # "vulnerability": "", + # "fixed_by_purl": PackageURL( + # type="maven", + # namespace="com.fasterxml.jackson.core", + # name="jackson-databind", + # version="2.13.2", + # qualifiers={}, + # subpath=None, + # ), + # "fixed_by_purl_vulnerabilities": [""], + # }, + # { + # "vulnerability": "", + # "fixed_by_purl": None, + # "fixed_by_purl_vulnerabilities": [], + # }, + # ], + # } + + print("\nfirst_vulnerable_package.purl = {}\n".format(first_vulnerable_package.purl)) + + print("\nfirst_vulnerable_package = {}\n".format(first_vulnerable_package)) + + assert ( + first_vulnerable_package.purl + == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" + ) + + # assert ( + # first_vulnerable_package + # # == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" + # == "" + # ) + + purl_string = "pkg:pypi/redis@4.1.1" + purl = PackageURL.from_string(purl_string) + purl_to_dict = purl.to_dict() + + # For namespace, version, qualifiers and subpath, we need to add the or * to avoid an IntegrityError, e.g., django.db.utils.IntegrityError: null value in column "subpath" violates not-null constraint + vulnerablecode_package = models.Package.objects.create( + type=purl_to_dict.get("type"), + namespace=purl_to_dict.get("namespace") or "", + name=purl_to_dict.get("name"), + version=purl_to_dict.get("version") or "", + qualifiers=purl_to_dict.get("qualifiers") or {}, + subpath=purl_to_dict.get("subpath") or "", + ) + + print("\nvulnerablecode_package = {}\n".format(vulnerablecode_package)) + print( + "\nvulnerablecode_package.fixed_package_details = {}\n".format( + vulnerablecode_package.fixed_package_details + ) + ) + + # =============================== + + # # Dictionary with class values + # my_dict = {"obj1": MyClass(1), "obj2": MyClass(2)} + + # # Print the dictionary + # print(my_dict) + + # assert vuln_packages.distinct()[0].fixed_package_details == purl_dict + + # banana = {'purl': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.13.1', qualifiers={}, subpath=None), 'closest_non_vulnerable': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.14.0-rc1', qualifiers={}, subpath=None), 'latest_non_vulnerable': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.14.0-rc1', qualifiers={}, subpath=None), 'vulnerabilities': [{'vulnerability': , 'fixed_by_purl': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.13.2', qualifiers={}, subpath=None), 'fixed_by_purl_vulnerabilities': []}, {'vulnerability': , 'fixed_by_purl': None, 'fixed_by_purl_vulnerabilities': []}]} + + # print('\nbanana = {}\n'.format(banana)) + + # assert vuln_packages.distinct()[0].`fixed_package_details` == banana + + print( + "\nvuln_packages.distinct()[0].fixed_package_details = {}\n".format( + vuln_packages.distinct()[0].fixed_package_details + ) + ) + + # print(vuln_packages.distinct()[0]["vulnerabilities"].fixed_package_details) # Error: TypeError: 'Package' object is not subscriptable + + print( + "\ntype(vuln_packages.distinct()[0].fixed_package_details) = {}\n".format( + type(vuln_packages.distinct()[0].fixed_package_details) + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details.get("purl") = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details.get("purl") + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details["purl"] = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details["purl"] + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"] = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"] + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0] = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0] + ) + ) + + print( + '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0]["vulnerability"] = {}\n'.format( + vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0][ + "vulnerability" + ] + ) + ) + + print("") def test_univers_version_comparisons(self): assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") @@ -405,113 +662,115 @@ def test_sort_by_version(self): assert sorted_pkgs[0].purl == "pkg:npm/sequelize@3.9.1" assert sorted_pkgs[-1].purl == "pkg:npm/sequelize@3.40.1" - def test_string_to_purl_to_dict_to_package(self): - # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., - # a . - - # Convert a PURL string to a PURL. - purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - purl = PackageURL.from_string(purl_string) - - assert type(purl) == PackageURL - assert purl.type == "maven" - assert purl.qualifiers == {} - assert purl.subpath == None - - # Convert the PURL to a dictionary. - # ALERT: 2023-08-15 Tuesday 13:18:09. What about using the function 'def purl_to_dict(purl: PackageURL)'? Confusingly similar name but it seems designed to address the issue raised here (and looks useful for passing the data to the Jinja2 template). - # It appears that this step is where the unwanted None values are created for qualifiers and - # subpath when the PURL does not already contain values for those attributes. - purl_to_dict = purl.to_dict() - - assert purl_to_dict == { - "type": "maven", - "namespace": "org.apache.tomcat.embed", - "name": "tomcat-embed-core", - "version": "9.0.31", - "qualifiers": None, - "subpath": None, - } - assert purl_to_dict.get("qualifiers") == None - assert purl_to_dict.get("subpath") == None - - # Convert the dictionary to a VulnerableCode Package, i.e., - # a - - # If subpath is None we get error: django.db.utils.IntegrityError: null value in column - # "subpath" violates not-null constraint -- need to convert value from None to empty string. - # Similar issue with qualifiers, which must be converted from None to {}. - - # I've structured the following in this way because trying instead to use - # "with pytest.raises(IntegrityError):" will throw the error - # django.db.transaction.TransactionManagementError: An error occurred in the current - # transaction. You can't execute queries until the end of the 'atomic' block. - - try: - with transaction.atomic(): - vulnerablecode_package = models.Package.objects.create( - type=purl_to_dict.get("type"), - namespace=purl_to_dict.get("namespace"), - name=purl_to_dict.get("name"), - version=purl_to_dict.get("version"), - qualifiers=purl_to_dict.get("qualifiers"), - subpath=purl_to_dict.get("subpath"), - ) - except IntegrityError: - print("\nAs expected, an IntegrityError has occurred.\n") - - # This will avoid the IntegrityError: - if purl_to_dict.get("qualifiers") is None: - purl_to_dict["qualifiers"] = {} - if purl_to_dict.get("subpath") is None: - purl_to_dict["subpath"] = "" - - # Check the qualifiers and subpath values again. - assert purl_to_dict.get("qualifiers") == {} - assert purl_to_dict.get("subpath") == "" - - vulnerablecode_package = models.Package.objects.create( - type=purl_to_dict.get("type"), - namespace=purl_to_dict.get("namespace"), - name=purl_to_dict.get("name"), - version=purl_to_dict.get("version"), - qualifiers=purl_to_dict.get("qualifiers"), - subpath=purl_to_dict.get("subpath"), - ) - - assert type(vulnerablecode_package) == models.Package - assert ( - vulnerablecode_package.purl - == "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - ) - assert vulnerablecode_package.qualifiers == {} - assert vulnerablecode_package.subpath == "" - - def test_compare_package_major_versions(self): - # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., - # a . - - # Convert a PURL string to a PURL. - purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - purl = PackageURL.from_string(purl_string) - - assert type(purl) == PackageURL - assert purl.type == "maven" - assert purl.qualifiers == {} - assert purl.subpath == None - - print("\npurl_string = {}".format(purl_string)) - - print("\npurl = {}".format(purl)) - - print("\nHello VulnerableCode!\n") - - all_packages = Package.objects - print("\nPackage.objects = {}\n".format(Package.objects)) - print("\nall_packages.distinct() = {}\n".format(all_packages.distinct())) - print("\nall_packages.distinct()[0] = {}\n".format(all_packages.distinct()[0])) - - for pkg in all_packages.distinct(): - print(PackageURL.from_string(pkg.purl)) - - print("") + # # ZAP: 2023-09-07 Thursday 20:05:40. This has served its purpose and can be removed after a last close look. + # def test_string_to_purl_to_dict_to_package(self): + # # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., + # # a . + + # # Convert a PURL string to a PURL. + # purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + # purl = PackageURL.from_string(purl_string) + + # assert type(purl) == PackageURL + # assert purl.type == "maven" + # assert purl.qualifiers == {} + # assert purl.subpath == None + + # # Convert the PURL to a dictionary. + # # ALERT: 2023-08-15 Tuesday 13:18:09. What about using the function 'def purl_to_dict(purl: PackageURL)'? Confusingly similar name but it seems designed to address the issue raised here (and looks useful for passing the data to the Jinja2 template). + # # It appears that this step is where the unwanted None values are created for qualifiers and + # # subpath when the PURL does not already contain values for those attributes. + # purl_to_dict = purl.to_dict() + + # assert purl_to_dict == { + # "type": "maven", + # "namespace": "org.apache.tomcat.embed", + # "name": "tomcat-embed-core", + # "version": "9.0.31", + # "qualifiers": None, + # "subpath": None, + # } + # assert purl_to_dict.get("qualifiers") == None + # assert purl_to_dict.get("subpath") == None + + # # Convert the dictionary to a VulnerableCode Package, i.e., + # # a + + # # If subpath is None we get error: django.db.utils.IntegrityError: null value in column + # # "subpath" violates not-null constraint -- need to convert value from None to empty string. + # # Similar issue with qualifiers, which must be converted from None to {}. + + # # I've structured the following in this way because trying instead to use + # # "with pytest.raises(IntegrityError):" will throw the error + # # django.db.transaction.TransactionManagementError: An error occurred in the current + # # transaction. You can't execute queries until the end of the 'atomic' block. + + # try: + # with transaction.atomic(): + # vulnerablecode_package = models.Package.objects.create( + # type=purl_to_dict.get("type"), + # namespace=purl_to_dict.get("namespace"), + # name=purl_to_dict.get("name"), + # version=purl_to_dict.get("version"), + # qualifiers=purl_to_dict.get("qualifiers"), + # subpath=purl_to_dict.get("subpath"), + # ) + # except IntegrityError: + # print("\nAs expected, an IntegrityError has occurred.\n") + + # # This will avoid the IntegrityError: + # if purl_to_dict.get("qualifiers") is None: + # purl_to_dict["qualifiers"] = {} + # if purl_to_dict.get("subpath") is None: + # purl_to_dict["subpath"] = "" + + # # Check the qualifiers and subpath values again. + # assert purl_to_dict.get("qualifiers") == {} + # assert purl_to_dict.get("subpath") == "" + + # vulnerablecode_package = models.Package.objects.create( + # type=purl_to_dict.get("type"), + # namespace=purl_to_dict.get("namespace"), + # name=purl_to_dict.get("name"), + # version=purl_to_dict.get("version"), + # qualifiers=purl_to_dict.get("qualifiers"), + # subpath=purl_to_dict.get("subpath"), + # ) + + # assert type(vulnerablecode_package) == models.Package + # assert ( + # vulnerablecode_package.purl + # == "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + # ) + # assert vulnerablecode_package.qualifiers == {} + # assert vulnerablecode_package.subpath == "" + + # # ZAP: 2023-09-07 Thursday 20:32:35. Ditch this, right? + # def test_compare_package_major_versions(self): + # # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., + # # a . + + # # Convert a PURL string to a PURL. + # purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" + # purl = PackageURL.from_string(purl_string) + + # assert type(purl) == PackageURL + # assert purl.type == "maven" + # assert purl.qualifiers == {} + # assert purl.subpath == None + + # print("\npurl_string = {}".format(purl_string)) + + # print("\npurl = {}".format(purl)) + + # print("\nHello VulnerableCode!\n") + + # all_packages = Package.objects + # print("\nPackage.objects = {}\n".format(Package.objects)) + # print("\nall_packages.distinct() = {}\n".format(all_packages.distinct())) + # print("\nall_packages.distinct()[0] = {}\n".format(all_packages.distinct()[0])) + + # for pkg in all_packages.distinct(): + # print(PackageURL.from_string(pkg.purl)) + + # print("") From 4314820def4df1d2f436c0a666813ce73452b4f9 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 11 Sep 2023 18:54:48 -0700 Subject: [PATCH 35/53] Finish package details code and template, refactor/create package-related tests #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 20 +- .../templates/package_details.html | 2 +- vulnerabilities/tests/test_models.py | 626 ++++-------------- 3 files changed, 149 insertions(+), 499 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 5c71a6bd8..3623191bf 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -761,14 +761,6 @@ def fixed_package_details(self): sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) closest_fixed_package = sort_fixed_by_packages_by_version[0] closest_fixed_package_vulns = closest_fixed_package.affected_by - print( - "\nclosest_fixed_package_vulns = {}\n".format(closest_fixed_package_vulns) - ) - print( - "\ntype(closest_fixed_package_vulns) = {}\n".format( - type(closest_fixed_package_vulns) - ) - ) else: closest_fixed_package = None @@ -790,13 +782,13 @@ def fixed_package_details(self): dict_vuln["fixed_by_purl_vulnerabilities"] = [] # Temporary print output during dev/testing. - from pprint import pprint + # from pprint import pprint - pprint( - purl_dict, - sort_dicts=False, - ) - print("") + # pprint( + # purl_dict, + # sort_dicts=False, + # ) + # print("") return purl_dict diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 9d9c70b77..6d610feb7 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -24,7 +24,7 @@ - @@ -65,7 +65,7 @@ {% if fixed_package_details.latest_non_vulnerable.version %} {{ fixed_package_details.latest_non_vulnerable.version }} {% else %} - None. + None. {% endif %} @@ -77,7 +77,7 @@
    - Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }}) + Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }})
    + ", - # "fixed_by_purl": PackageURL( - # type="maven", - # namespace="com.fasterxml.jackson.core", - # name="jackson-databind", - # version="2.13.2", - # qualifiers={}, - # subpath=None, - # ), - # "fixed_by_purl_vulnerabilities": [""], - # }, - # { - # "vulnerability": "", - # "fixed_by_purl": None, - # "fixed_by_purl_vulnerabilities": [], - # }, - # ], - # } - - print("\nfirst_vulnerable_package.purl = {}\n".format(first_vulnerable_package.purl)) - - print("\nfirst_vulnerable_package = {}\n".format(first_vulnerable_package)) - - assert ( - first_vulnerable_package.purl - == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" - ) + assert first_fixed_by_package.purl == "pkg:pypi/redis@4.3.6" - # assert ( - # first_vulnerable_package - # # == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1" - # == "" - # ) + def test_string_to_package(self): - purl_string = "pkg:pypi/redis@4.1.1" + purl_string = "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" purl = PackageURL.from_string(purl_string) purl_to_dict = purl.to_dict() @@ -513,77 +343,18 @@ def test_get_vulnerable_packages(self): subpath=purl_to_dict.get("subpath") or "", ) - print("\nvulnerablecode_package = {}\n".format(vulnerablecode_package)) - print( - "\nvulnerablecode_package.fixed_package_details = {}\n".format( - vulnerablecode_package.fixed_package_details - ) - ) - - # =============================== - - # # Dictionary with class values - # my_dict = {"obj1": MyClass(1), "obj2": MyClass(2)} - - # # Print the dictionary - # print(my_dict) - - # assert vuln_packages.distinct()[0].fixed_package_details == purl_dict - - # banana = {'purl': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.13.1', qualifiers={}, subpath=None), 'closest_non_vulnerable': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.14.0-rc1', qualifiers={}, subpath=None), 'latest_non_vulnerable': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.14.0-rc1', qualifiers={}, subpath=None), 'vulnerabilities': [{'vulnerability': , 'fixed_by_purl': PackageURL(type='maven', namespace='com.fasterxml.jackson.core', name='jackson-databind', version='2.13.2', qualifiers={}, subpath=None), 'fixed_by_purl_vulnerabilities': []}, {'vulnerability': , 'fixed_by_purl': None, 'fixed_by_purl_vulnerabilities': []}]} - - # print('\nbanana = {}\n'.format(banana)) - - # assert vuln_packages.distinct()[0].`fixed_package_details` == banana - - print( - "\nvuln_packages.distinct()[0].fixed_package_details = {}\n".format( - vuln_packages.distinct()[0].fixed_package_details - ) - ) - - # print(vuln_packages.distinct()[0]["vulnerabilities"].fixed_package_details) # Error: TypeError: 'Package' object is not subscriptable - - print( - "\ntype(vuln_packages.distinct()[0].fixed_package_details) = {}\n".format( - type(vuln_packages.distinct()[0].fixed_package_details) - ) - ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details.get("purl") = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details.get("purl") - ) - ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details["purl"] = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details["purl"] - ) - ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"] = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"] - ) - ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0] = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0] - ) + assert type(vulnerablecode_package) == models.Package + assert vulnerablecode_package.purl == "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" + assert vulnerablecode_package.package_url == "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" + assert ( + vulnerablecode_package.plain_package_url + == "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" ) - - print( - '\nvuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0]["vulnerability"] = {}\n'.format( - vuln_packages.distinct()[0].fixed_package_details["vulnerabilities"][0][ - "vulnerability" - ] - ) + assert ( + vulnerablecode_package.get_absolute_url() + == "/packages/pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" ) - print("") - def test_univers_version_comparisons(self): assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") @@ -661,116 +432,3 @@ def test_sort_by_version(self): assert sorted_pkgs[0].purl == "pkg:npm/sequelize@3.9.1" assert sorted_pkgs[-1].purl == "pkg:npm/sequelize@3.40.1" - - # # ZAP: 2023-09-07 Thursday 20:05:40. This has served its purpose and can be removed after a last close look. - # def test_string_to_purl_to_dict_to_package(self): - # # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., - # # a . - - # # Convert a PURL string to a PURL. - # purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - # purl = PackageURL.from_string(purl_string) - - # assert type(purl) == PackageURL - # assert purl.type == "maven" - # assert purl.qualifiers == {} - # assert purl.subpath == None - - # # Convert the PURL to a dictionary. - # # ALERT: 2023-08-15 Tuesday 13:18:09. What about using the function 'def purl_to_dict(purl: PackageURL)'? Confusingly similar name but it seems designed to address the issue raised here (and looks useful for passing the data to the Jinja2 template). - # # It appears that this step is where the unwanted None values are created for qualifiers and - # # subpath when the PURL does not already contain values for those attributes. - # purl_to_dict = purl.to_dict() - - # assert purl_to_dict == { - # "type": "maven", - # "namespace": "org.apache.tomcat.embed", - # "name": "tomcat-embed-core", - # "version": "9.0.31", - # "qualifiers": None, - # "subpath": None, - # } - # assert purl_to_dict.get("qualifiers") == None - # assert purl_to_dict.get("subpath") == None - - # # Convert the dictionary to a VulnerableCode Package, i.e., - # # a - - # # If subpath is None we get error: django.db.utils.IntegrityError: null value in column - # # "subpath" violates not-null constraint -- need to convert value from None to empty string. - # # Similar issue with qualifiers, which must be converted from None to {}. - - # # I've structured the following in this way because trying instead to use - # # "with pytest.raises(IntegrityError):" will throw the error - # # django.db.transaction.TransactionManagementError: An error occurred in the current - # # transaction. You can't execute queries until the end of the 'atomic' block. - - # try: - # with transaction.atomic(): - # vulnerablecode_package = models.Package.objects.create( - # type=purl_to_dict.get("type"), - # namespace=purl_to_dict.get("namespace"), - # name=purl_to_dict.get("name"), - # version=purl_to_dict.get("version"), - # qualifiers=purl_to_dict.get("qualifiers"), - # subpath=purl_to_dict.get("subpath"), - # ) - # except IntegrityError: - # print("\nAs expected, an IntegrityError has occurred.\n") - - # # This will avoid the IntegrityError: - # if purl_to_dict.get("qualifiers") is None: - # purl_to_dict["qualifiers"] = {} - # if purl_to_dict.get("subpath") is None: - # purl_to_dict["subpath"] = "" - - # # Check the qualifiers and subpath values again. - # assert purl_to_dict.get("qualifiers") == {} - # assert purl_to_dict.get("subpath") == "" - - # vulnerablecode_package = models.Package.objects.create( - # type=purl_to_dict.get("type"), - # namespace=purl_to_dict.get("namespace"), - # name=purl_to_dict.get("name"), - # version=purl_to_dict.get("version"), - # qualifiers=purl_to_dict.get("qualifiers"), - # subpath=purl_to_dict.get("subpath"), - # ) - - # assert type(vulnerablecode_package) == models.Package - # assert ( - # vulnerablecode_package.purl - # == "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - # ) - # assert vulnerablecode_package.qualifiers == {} - # assert vulnerablecode_package.subpath == "" - - # # ZAP: 2023-09-07 Thursday 20:32:35. Ditch this, right? - # def test_compare_package_major_versions(self): - # # Convert a PURL string to a PURL to a dictionary to a VulnerableCode Package, i.e., - # # a . - - # # Convert a PURL string to a PURL. - # purl_string = "pkg:maven/org.apache.tomcat.embed/tomcat-embed-core@9.0.31" - # purl = PackageURL.from_string(purl_string) - - # assert type(purl) == PackageURL - # assert purl.type == "maven" - # assert purl.qualifiers == {} - # assert purl.subpath == None - - # print("\npurl_string = {}".format(purl_string)) - - # print("\npurl = {}".format(purl)) - - # print("\nHello VulnerableCode!\n") - - # all_packages = Package.objects - # print("\nPackage.objects = {}\n".format(Package.objects)) - # print("\nall_packages.distinct() = {}\n".format(all_packages.distinct())) - # print("\nall_packages.distinct()[0] = {}\n".format(all_packages.distinct()[0])) - - # for pkg in all_packages.distinct(): - # print(PackageURL.from_string(pkg.purl)) - - # print("") From df7f4049b08fd8e1eae502c2899ccb921260c492 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 18 Sep 2023 12:17:39 -0700 Subject: [PATCH 36/53] Commit the initial refactoring changes from last week #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/api.py | 160 ++++++++++++++++++++++++++- vulnerabilities/models.py | 120 ++++++++++---------- vulnerabilities/tests/test_models.py | 5 +- 3 files changed, 221 insertions(+), 64 deletions(-) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 6045fc8ca..a774ff0b6 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -48,11 +48,87 @@ class MinimalPackageSerializer(serializers.HyperlinkedModelSerializer): Used for nesting inside vulnerability focused APIs. """ + affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities") + + def get_affected_vulnerabilities(self, package): + # HOT: 2023-09-13 Wednesday 13:48:01. 'parent_affected_vulnerabilities' identifies (1) the vulnerabilities (if any) that affect the given package that itself fixes one of the target purl's vulnerabilities, and (2) + # Ex: + # package.purl = pkg:pypi/redis@4.3.6 + # parent_affected_vulnerabilities = [{'vulnerability': , 'fixed_by_purl': PackageURL(type='pypi', namespace=None, name='redis', version='5.0.0b1', qualifiers={}, subpath=None), 'fixed_by_purl_vulnerabilities': []}] + + parent_affected_vulnerabilities = package.fixed_package_details.get("vulnerabilities", None) + affected_vulnerabilities = [] + + print("\npackage.purl = {}".format(package.purl)) + print("\nparent_affected_vulnerabilities = {}\n".format(parent_affected_vulnerabilities)) + # print( + # "\nparent_affected_vulnerabilities.get('vulnerability', None) = {}\n".format( + # parent_affected_vulnerabilities.get("vulnerability", None) + # ) + # ) + # print( + # "\nparent_affected_vulnerabilities.get('fixed_by_purl', None) = {}\n".format( + # parent_affected_vulnerabilities.get("fixed_by_purl", None) + # ) + # ) + # print( + # "\nparent_affected_vulnerabilities.get('fixed_by_purl_vulnerabilities', None) = {}\n".format( + # parent_affected_vulnerabilities.get("fixed_by_purl_vulnerabilities", None) + # ) + # ) + + if parent_affected_vulnerabilities: + for vuln in parent_affected_vulnerabilities: + affected_vulnerability = {} + + print( + '\nvuln.get("vulnerability", None) = {}'.format(vuln.get("vulnerability", None)) + ) + affected_vulnerability["vulnerability"] = vuln.get( + "vulnerability", None + ).vulnerability_id + + print( + '\nvuln.get("fixed_by_purl", None) = {}'.format(vuln.get("fixed_by_purl", None)) + ) + affected_vulnerability["fixed_by_purl"] = vuln.get( + "fixed_by_purl", None + ).to_string() + + print( + '\nvuln.get("fixed_by_purl_vulnerabilities", None) = {}'.format( + vuln.get("fixed_by_purl_vulnerabilities", None) + ) + ) + affected_vulnerability["fixed_by_purl_vulnerabilities"] = [ + fixed_by_purl_vulnerability.vulnerability_id + for fixed_by_purl_vulnerability in vuln.get( + "fixed_by_purl_vulnerabilities", None + ) + ] + + affected_vulnerabilities.append(affected_vulnerability) + + print("affected_vulnerability = {}".format(affected_vulnerability)) + print("affected_vulnerabilities = {}".format(affected_vulnerabilities)) + + return affected_vulnerabilities + + # ZAP: 2023-09-12 Tuesday 19:12:22. Not what we want but this does return the purl, which we can use to find all of its own vulns (if any). + # return package.purl + + # def get_affected_vulnerabilities_saved(self, package) -> dict: + # """ + # Return a mapping of vulnerabilities that affects the given `package`. + # """ + # # return self.get_vulnerabilities_for_a_package(package=package, fix=False) + # return self.get_vulnerabilities_for_a_package(package=self, fix=False) + purl = serializers.CharField(source="package_url") class Meta: model = Package - fields = ["url", "purl", "is_vulnerable"] + fields = ["url", "purl", "is_vulnerable", "affected_by_vulnerabilities"] class MinimalVulnerabilitySerializer(serializers.HyperlinkedModelSerializer): @@ -93,9 +169,41 @@ def to_representation(self, instance): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") + # ZAP: 2023-09-14 Thursday 11:34:26. Add closest fixed package from the Package UI here. + + closest_fixed_package = serializers.SerializerMethodField("get_closest_fixed_package") + + def get_closest_fixed_package(self, instance): + return "Coming soon!" + + # test_data = super().to_representation(instance) + # vulnerabilities = [vuln["vulnerability"] for vuln in test_data["vulnerabilities"]] + # test_data["vulnerabilities"] = vulnerabilities + + # return test_data + + # # ZAP: From the PackageSerializer class: + # # next_non_vulnerable_version = serializers.SerializerMethodField("get_closest_non_vulnerable") + + # def get_closest_non_vulnerable(self, package): + # closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) + # if closest_non_vulnerable: + # return closest_non_vulnerable.version + # else: + # return None + class Meta: model = Vulnerability - fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] + # fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] + fields = [ + "url", + "vulnerability_id", + "summary", + "references", + "fixed_packages", + "closest_fixed_package", + "aliases", + ] class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer): @@ -108,8 +216,34 @@ class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") + # ALERT: 2023-09-14 Thursday 15:16:05. For some reason, favorite_color is not appearing in the output. + # favorite_color = serializers.SerializerMethodField("get_favorite_color") + + # # def get_closest_non_vulnerable(self, package): + # def get_favorite_color(self, package): + # # closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) + # # if closest_non_vulnerable: + # # return closest_non_vulnerable.version + # # else: + # # return None + # return "red!" + + favorite_color = MinimalPackageSerializer( + many=True, source="filtered_fixed_packages", read_only=True + ) + class Meta: model = Vulnerability + # fields = [ + # "url", + # "vulnerability_id", + # "summary", + # "aliases", + # "fixed_packages", + # "affected_packages", + # "references", + # ] + fields = [ "url", "vulnerability_id", @@ -118,6 +252,7 @@ class Meta: "fixed_packages", "affected_packages", "references", + "favorite_color", ] @@ -126,6 +261,24 @@ class PackageSerializer(serializers.HyperlinkedModelSerializer): Lookup software package using Package URLs """ + next_non_vulnerable_version = serializers.SerializerMethodField("get_closest_non_vulnerable") + + def get_closest_non_vulnerable(self, package): + closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) + if closest_non_vulnerable: + return closest_non_vulnerable.version + else: + return None + + latest_non_vulnerable_version = serializers.SerializerMethodField("get_latest_non_vulnerable") + + def get_latest_non_vulnerable(self, package): + latest_non_vulnerable = package.fixed_package_details.get("latest_non_vulnerable", None) + if latest_non_vulnerable: + return latest_non_vulnerable.version + else: + return None + purl = serializers.CharField(source="package_url") affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities") @@ -161,6 +314,7 @@ def get_vulnerabilities_for_a_package(self, package, fix) -> dict: to_attr="filtered_fixed_packages", ) ) + # so this calls VulnSerializerRefsAndSummary return VulnSerializerRefsAndSummary( instance=qs, many=True, @@ -192,6 +346,8 @@ class Meta: "subpath", "affected_by_vulnerabilities", "fixing_vulnerabilities", + "next_non_vulnerable_version", + "latest_non_vulnerable_version", ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 3623191bf..66a25d566 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -645,19 +645,17 @@ def get_absolute_url(self): """ return reverse("package_details", args=[self.purl]) - def get_fixed_packages(self, package, fix=True): + def get_fixing_package_versions(self, fix=True): """ - With fix=True, return a queryset of all packages that fix a vulnerability and have the - same type, namespace, name, subpath and qualifiers as the `package`. - With fix=False, return a queryset of all packages that have the same type, namespace, - name, subpath and qualifiers as the `package`, whether or not they fix any vulnerability. + Return a queryset of all the package versions of this `package` that fix any vulnerability. + If `fix` is False, return all package versions whether or not they fix a vulnerability. """ filter_dict = { - "name": package.name, - "namespace": package.namespace, - "type": package.type, - "qualifiers": package.qualifiers, - "subpath": package.subpath, + "name": self.name, + "namespace": self.namespace, + "type": self.type, + "qualifiers": self.qualifiers, + "subpath": self.subpath, } if fix: @@ -665,15 +663,14 @@ def get_fixed_packages(self, package, fix=True): return Package.objects.filter(**filter_dict).distinct() - def sort_by_version(self, packages_to_sort): + def sort_by_version(self, packages): """ - Use the univers version_class to sort the incoming list of vulnerabilities.models.Package - objects. + Return a list of `packages` sorted by version. """ - univers_version = RANGE_CLASS_BY_SCHEMES[packages_to_sort[0].type].version_class + version_class = RANGE_CLASS_BY_SCHEMES[packages[0].type].version_class return sorted( - packages_to_sort, - key=lambda x: univers_version(x.version), + packages, + key=lambda x: version_class(x.version), ) @property @@ -683,60 +680,61 @@ def fixed_package_details(self): package's vulnerabilities as well as the closest non-vulnerable version and the most recent non-vulnerable version. """ - fixed_packages = self.get_fixed_packages(package=self, fix=True) - qs = self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - all_sibling_packages = self.get_fixed_packages(package=self, fix=False) - - non_vuln_sibs = [] - for sib in all_sibling_packages: - if sib.is_vulnerable is False: - non_vuln_sibs.append(sib) + fixing_packages = self.get_fixing_package_versions(fix=True) + package_vulnerabilities = self.vulnerabilities.filter( + packagerelatedvulnerability__fix=False + ) + package_versions = self.get_fixing_package_versions(fix=False) - univers_version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class - later_non_vuln_sibs = [] + non_vulnerable_versions = [] + for version in package_versions: + if not version.is_vulnerable: + non_vulnerable_versions.append(version) - for non_vuln_sib in non_vuln_sibs: - non_vuln_sib_univers_version = univers_version_class( - PackageURL.from_string(non_vuln_sib.purl).version - ) + version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class + later_non_vulnerable_versions = [] - if univers_version_class(non_vuln_sib.version) > univers_version_class(self.version): - later_non_vuln_sibs.append(non_vuln_sib) + current_version = version_class(self.version) + for non_vuln_ver in non_vulnerable_versions: + if version_class(non_vuln_ver.version) > current_version: + later_non_vulnerable_versions.append(non_vuln_ver) - qs = qs.prefetch_related( + package_vulnerabilities = package_vulnerabilities.prefetch_related( Prefetch( "packages", - queryset=fixed_packages, + queryset=fixing_packages, to_attr="fixed_packages", ) ) - purl_dict = {} - purl_dict["purl"] = PackageURL.from_string(self.purl) + package_details = {} + package_details["purl"] = PackageURL.from_string(self.purl) - closest_non_vulnerable_sib = "" - latest_non_vulnerable_sib = "" + closest_non_vulnerable_version = "" + latest_non_vulnerable_version = "" - if len(later_non_vuln_sibs) > 0: - closest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[0] - latest_non_vulnerable_sib = self.sort_by_version(later_non_vuln_sibs)[-1] + # ZAP: 2023-09-18 Monday 11:40:05. I think Philippe and I got up to this point during last Thursday's code review. We also reviewed much of the API structure and output which still needs more updating -- and then some tests. - purl_dict["closest_non_vulnerable"] = PackageURL.from_string( - closest_non_vulnerable_sib.purl + if later_non_vulnerable_versions: + closest_non_vulnerable_version = self.sort_by_version(later_non_vulnerable_versions)[0] + latest_non_vulnerable_version = self.sort_by_version(later_non_vulnerable_versions)[-1] + + package_details["closest_non_vulnerable"] = PackageURL.from_string( + closest_non_vulnerable_version.purl ) - purl_dict["latest_non_vulnerable"] = PackageURL.from_string( - latest_non_vulnerable_sib.purl + package_details["latest_non_vulnerable"] = PackageURL.from_string( + latest_non_vulnerable_version.purl ) else: - purl_dict["closest_non_vulnerable"] = None - purl_dict["latest_non_vulnerable"] = None + package_details["closest_non_vulnerable"] = None + package_details["latest_non_vulnerable"] = None - purl_dict.update({"vulnerabilities": []}) + package_details.update({"vulnerabilities": []}) - for vuln in qs: - purl_dict["vulnerabilities"].append({"vulnerability": vuln}) + for vuln in package_vulnerabilities: + package_details["vulnerabilities"].append({"vulnerability": vuln}) closest_fixed_package = "" closest_fixed_package_vulns = "" @@ -747,13 +745,13 @@ def fixed_package_details(self): if len(vuln_fixed_packages) > 0: for fixed_pkg in vuln_fixed_packages: - fixed_pkg_univers_version = univers_version_class( + fixed_pkg_univers_version = version_class( PackageURL.from_string(fixed_pkg.purl).version ) - if fixed_pkg in fixed_packages and univers_version_class( + if fixed_pkg in fixing_packages and version_class( fixed_pkg.version - ) > univers_version_class(self.version): + ) > version_class(self.version): later_fixed_packages.append(fixed_pkg) @@ -766,7 +764,7 @@ def fixed_package_details(self): closest_fixed_package = None closest_fixed_package_vulns = None - for dict_vuln in purl_dict["vulnerabilities"]: + for dict_vuln in package_details["vulnerabilities"]: if dict_vuln["vulnerability"] == vuln: if closest_fixed_package: @@ -782,15 +780,15 @@ def fixed_package_details(self): dict_vuln["fixed_by_purl_vulnerabilities"] = [] # Temporary print output during dev/testing. - # from pprint import pprint + from pprint import pprint - # pprint( - # purl_dict, - # sort_dicts=False, - # ) - # print("") + pprint( + package_details, + sort_dicts=False, + ) + print("") - return purl_dict + return package_details class PackageRelatedVulnerability(models.Model): diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index a35cef7c8..2f33a2d6e 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -318,7 +318,10 @@ def test_get_vulnerable_packages(self): assert vuln_packages.distinct().count() == 2 first_vulnerable_package = vuln_packages.distinct()[0] - matching_fixed_packages = first_vulnerable_package.get_fixed_packages( + # matching_fixed_packages = first_vulnerable_package.get_fixed_packages( + # first_vulnerable_package + # ) + matching_fixed_packages = first_vulnerable_package.get_fixing_package_versions( first_vulnerable_package ) first_fixed_by_package = matching_fixed_packages[0] From e475e5d19fd2e7cc980bfe6b6171da820eed69d4 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Thu, 21 Sep 2023 12:44:44 -0700 Subject: [PATCH 37/53] Refactor package details-related code #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/models.py | 178 +++++++++--------- .../templates/package_details.html | 24 +-- vulnerabilities/templates/packages.html | 6 +- .../templates/vulnerabilities.html | 4 +- .../templates/vulnerability_details.html | 16 +- vulnerabilities/tests/test_models.py | 30 ++- vulnerablecode/static/css/custom.css | 32 ++++ 7 files changed, 153 insertions(+), 137 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 66a25d566..75b834c71 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -604,7 +604,6 @@ def __str__(self): return self.package_url @property - # TODO: consider renaming to "affected_by" def affected_by(self): """ Return a queryset of vulnerabilities affecting this package. @@ -645,7 +644,8 @@ def get_absolute_url(self): """ return reverse("package_details", args=[self.purl]) - def get_fixing_package_versions(self, fix=True): + # TODO: There are other methods, variables etc. in models.py that need similar renaming. + def get_fixed_by_package_versions(self, fix=True): """ Return a queryset of all the package versions of this `package` that fix any vulnerability. If `fix` is False, return all package versions whether or not they fix a vulnerability. @@ -667,128 +667,118 @@ def sort_by_version(self, packages): """ Return a list of `packages` sorted by version. """ + if not packages: + return [] + version_class = RANGE_CLASS_BY_SCHEMES[packages[0].type].version_class return sorted( packages, key=lambda x: version_class(x.version), ) + @property + def version_class(self): + return RANGE_CLASS_BY_SCHEMES[self.type].version_class + + @property + def current_version(self): + return self.version_class(self.version) + @property def fixed_package_details(self): """ - For a given package, report the closest subsequent fixed by version for each of that - package's vulnerabilities as well as the closest non-vulnerable version and the most recent - non-vulnerable version. + Return a mapping of vulnerabilities that affect this package and the closest and + latest non-vulnerable versions. """ - fixing_packages = self.get_fixing_package_versions(fix=True) - package_vulnerabilities = self.vulnerabilities.filter( - packagerelatedvulnerability__fix=False - ) - package_versions = self.get_fixing_package_versions(fix=False) + package_details = {} + package_details["purl"] = PackageURL.from_string(self.purl) + + closest_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() + package_details["closest_non_vulnerable"] = closest_non_vulnerable + package_details["latest_non_vulnerable"] = latest_non_vulnerable + + package_details["vulnerabilities"] = self.get_affecting_vulnerabilities() + + return package_details + + def get_non_vulnerable_versions(self): + """ + Return a tuple of the closest and latest non-vulnerable versions as PackageURLs. Return a tuple of + (None, None) if there is no non-vulnerable version. + """ + package_versions = self.get_fixed_by_package_versions(fix=False) non_vulnerable_versions = [] for version in package_versions: if not version.is_vulnerable: non_vulnerable_versions.append(version) - version_class = RANGE_CLASS_BY_SCHEMES[self.type].version_class later_non_vulnerable_versions = [] - - current_version = version_class(self.version) for non_vuln_ver in non_vulnerable_versions: - if version_class(non_vuln_ver.version) > current_version: + if self.version_class(non_vuln_ver.version) > self.current_version: later_non_vulnerable_versions.append(non_vuln_ver) - package_vulnerabilities = package_vulnerabilities.prefetch_related( - Prefetch( - "packages", - queryset=fixing_packages, - to_attr="fixed_packages", - ) - ) - - package_details = {} - package_details["purl"] = PackageURL.from_string(self.purl) - - closest_non_vulnerable_version = "" - latest_non_vulnerable_version = "" + if later_non_vulnerable_versions: + sorted_versions = self.sort_by_version(later_non_vulnerable_versions) + closest_non_vulnerable_version = sorted_versions[0] + latest_non_vulnerable_version = sorted_versions[-1] - # ZAP: 2023-09-18 Monday 11:40:05. I think Philippe and I got up to this point during last Thursday's code review. We also reviewed much of the API structure and output which still needs more updating -- and then some tests. + closest_non_vulnerable = PackageURL.from_string(closest_non_vulnerable_version.purl) + latest_non_vulnerable = PackageURL.from_string(latest_non_vulnerable_version.purl) - if later_non_vulnerable_versions: - closest_non_vulnerable_version = self.sort_by_version(later_non_vulnerable_versions)[0] - latest_non_vulnerable_version = self.sort_by_version(later_non_vulnerable_versions)[-1] + return closest_non_vulnerable, latest_non_vulnerable - package_details["closest_non_vulnerable"] = PackageURL.from_string( - closest_non_vulnerable_version.purl - ) - package_details["latest_non_vulnerable"] = PackageURL.from_string( - latest_non_vulnerable_version.purl - ) + return None, None - else: + def get_affecting_vulnerabilities(self): + """ + Return a list of vulnerabilities that affect this package together with information regarding + the versions that fix the vulnerabilities. + """ + package_details_vulns = [] - package_details["closest_non_vulnerable"] = None - package_details["latest_non_vulnerable"] = None + fixed_by_packages = self.get_fixed_by_package_versions(fix=True) - package_details.update({"vulnerabilities": []}) + package_vulnerabilities = self.vulnerabilities.filter( + packagerelatedvulnerability__fix=False + ).prefetch_related( + Prefetch( + "packages", + queryset=fixed_by_packages, + to_attr="fixed_packages", + ) + ) for vuln in package_vulnerabilities: - package_details["vulnerabilities"].append({"vulnerability": vuln}) - - closest_fixed_package = "" - closest_fixed_package_vulns = "" + package_details_vulns.append({"vulnerability": vuln}) later_fixed_packages = [] - sort_fixed_by_packages_by_version = [] - vuln_fixed_packages = vuln.fixed_packages - if len(vuln_fixed_packages) > 0: - for fixed_pkg in vuln_fixed_packages: - - fixed_pkg_univers_version = version_class( - PackageURL.from_string(fixed_pkg.purl).version - ) - - if fixed_pkg in fixing_packages and version_class( - fixed_pkg.version - ) > version_class(self.version): - - later_fixed_packages.append(fixed_pkg) - - if later_fixed_packages: - sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) - closest_fixed_package = sort_fixed_by_packages_by_version[0] - closest_fixed_package_vulns = closest_fixed_package.affected_by - - else: - closest_fixed_package = None - closest_fixed_package_vulns = None - - for dict_vuln in package_details["vulnerabilities"]: - if dict_vuln["vulnerability"] == vuln: - - if closest_fixed_package: - dict_vuln["fixed_by_purl"] = PackageURL.from_string( - closest_fixed_package.purl - ) - - dict_vuln["fixed_by_purl_vulnerabilities"] = [ - fixed_pkg_vuln for fixed_pkg_vuln in closest_fixed_package_vulns - ] - else: - dict_vuln["fixed_by_purl"] = None - dict_vuln["fixed_by_purl_vulnerabilities"] = [] - - # Temporary print output during dev/testing. - from pprint import pprint - - pprint( - package_details, - sort_dicts=False, - ) - print("") - - return package_details + for fixed_pkg in vuln.fixed_packages: + if fixed_pkg not in fixed_by_packages: + continue + fixed_version = self.version_class(fixed_pkg.version) + if fixed_version > self.current_version: + later_fixed_packages.append(fixed_pkg) + + closest_fixed_package = None + closest_fixed_package_vulns = [] + + if later_fixed_packages: + sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) + closest_fixed_package = sort_fixed_by_packages_by_version[0] + fixed_by_purl = PackageURL.from_string(closest_fixed_package.purl) + closest_fixed_package_vulns = list(closest_fixed_package.affected_by) + + for vuln_details in package_details_vulns: + if vuln_details["vulnerability"] != vuln: + continue + vuln_details["fixed_by_purl"] = None + vuln_details["fixed_by_purl_vulnerabilities"] = [] + if closest_fixed_package: + vuln_details["fixed_by_purl"] = fixed_by_purl + vuln_details["fixed_by_purl_vulnerabilities"] = closest_fixed_package_vulns + + return package_details_vulns class PackageRelatedVulnerability(models.Model): diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 6d610feb7..246b3733e 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -53,7 +53,7 @@ {% if fixed_package_details.closest_non_vulnerable.version %} {{ fixed_package_details.closest_non_vulnerable.version }} {% else %} - None. + None. {% endif %}
    @@ -85,7 +85,7 @@ - + @@ -117,20 +117,20 @@ {% for vuln in value %} {% if vuln.vulnerability.vulnerability_id == vulnerability.vulnerability_id %} {% if vuln.fixed_by_purl is None %} - There are no reported fixed versions. + There are no reported fixed versions. {% else %} {% if vuln.fixed_by_purl_vulnerabilities|length == 0 %} {{ vuln.fixed_by_purl.version }}
    - Affected by 0 other vulnerabilities. + Affected by 0 other vulnerabilities. {% else %} {{ vuln.fixed_by_purl.version }} {% if vuln.fixed_by_purl_vulnerabilities|length != 1 %}
    - Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. {% else %}
    - Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerability. + Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerability. {% endif %}
    @@ -141,7 +141,7 @@
    - This version is affected by these other vulnerabilities: + This version is affected by these other vulnerabilities:
    {% for fixed_by_vuln in vuln.fixed_by_purl_vulnerabilities %}
    @@ -166,7 +166,7 @@ {% empty %}
    {% endfor %} @@ -176,7 +176,7 @@
    - Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }}) + Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }})
    Vulnerability SummaryFixed byFixed by
    - This package is not known to be affected by vulnerabilities. + This package is not known to be affected by vulnerabilities.
    @@ -211,7 +211,7 @@ {% empty %} {% endfor %} diff --git a/vulnerabilities/templates/packages.html b/vulnerabilities/templates/packages.html index 000fe45be..357d60ce7 100644 --- a/vulnerabilities/templates/packages.html +++ b/vulnerabilities/templates/packages.html @@ -24,7 +24,7 @@ - +
    - This package is not known to fix vulnerabilities. + This package is not known to fix vulnerabilities.
    @@ -41,14 +41,14 @@ - Affected by vulnerabilities + Affected by vulnerabilities diff --git a/vulnerabilities/templates/vulnerabilities.html b/vulnerabilities/templates/vulnerabilities.html index 023d3f97f..bdada6ee1 100644 --- a/vulnerabilities/templates/vulnerabilities.html +++ b/vulnerabilities/templates/vulnerabilities.html @@ -32,8 +32,8 @@ - - + + diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index 4d1af2e5c..c42f6d885 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -34,14 +34,14 @@
  • - Fixed by packages ({{ fixed_by_packages|length }}) + Fixed by packages ({{ fixed_by_packages|length }})
  • - Affected packages ({{ affected_packages|length }}) + Affected packages ({{ affected_packages|length }})
    • @@ -128,7 +128,7 @@
    - Fixed by packages ({{ fixed_by_packages|length }}) + Fixed by packages ({{ fixed_by_packages|length }})
    - Fixing vulnerabilities + Fixed by vulnerabilities
    Vulnerability id AliasesAffected packagesFixed by packagesAffected packagesFixed by packages
    @@ -142,14 +142,14 @@ {% empty %} {% endfor %} {% if fixed_by_packages|length > 3 %} {% endif %} @@ -157,7 +157,7 @@
    - Affected packages ({{ affected_packages|length }}) + Affected packages ({{ affected_packages|length }})
    - There are no known fixed packages. + There are no known fixed by packages.
    - See Fixed by packages tab for more + See Fixed by packages tab for more
    @@ -171,14 +171,14 @@ {% empty %} {% endfor %} {% if affected_packages|length > 3 %} {% endif %} diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 2f33a2d6e..bb9673e9c 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -233,22 +233,16 @@ def test_fixed_package_details(self): searched_for_package_details = searched_for_package.fixed_package_details - purl_dict = { + package_details = { "purl": PackageURL( type="pypi", - namespace=None, name="redis", version="4.1.1", - qualifiers={}, - subpath=None, ), "closest_non_vulnerable": PackageURL( type="pypi", - namespace=None, name="redis", version="5.0.0b1", - qualifiers={}, - subpath=None, ), "latest_non_vulnerable": PackageURL( type="pypi", @@ -286,7 +280,7 @@ def test_fixed_package_details(self): ], } - assert searched_for_package_details == purl_dict + assert searched_for_package_details == package_details assert searched_for_package_details.get("closest_non_vulnerable") == PackageURL( type="pypi", @@ -306,11 +300,11 @@ def test_fixed_package_details(self): subpath=None, ) - qs_searched_for_package_fixing = searched_for_package.fixing - assert type(qs_searched_for_package_fixing) == models.VulnerabilityQuerySet - assert qs_searched_for_package_fixing.count() == 0 - assert len(qs_searched_for_package_fixing) == 0 - assert list(qs_searched_for_package_fixing) == [] + searched_for_package_fixing = searched_for_package.fixing + assert type(searched_for_package_fixing) == models.VulnerabilityQuerySet + assert searched_for_package_fixing.count() == 0 + assert len(searched_for_package_fixing) == 0 + assert list(searched_for_package_fixing) == [] def test_get_vulnerable_packages(self): vuln_packages = Package.objects.vulnerable() @@ -318,10 +312,8 @@ def test_get_vulnerable_packages(self): assert vuln_packages.distinct().count() == 2 first_vulnerable_package = vuln_packages.distinct()[0] - # matching_fixed_packages = first_vulnerable_package.get_fixed_packages( - # first_vulnerable_package - # ) - matching_fixed_packages = first_vulnerable_package.get_fixing_package_versions( + + matching_fixed_packages = first_vulnerable_package.get_fixed_by_package_versions( first_vulnerable_package ) first_fixed_by_package = matching_fixed_packages[0] @@ -336,7 +328,9 @@ def test_string_to_package(self): purl = PackageURL.from_string(purl_string) purl_to_dict = purl.to_dict() - # For namespace, version, qualifiers and subpath, we need to add the or * to avoid an IntegrityError, e.g., django.db.utils.IntegrityError: null value in column "subpath" violates not-null constraint + # For namespace, version, qualifiers and subpath, we need to add the or * to avoid an + # IntegrityError, e.g., django.db.utils.IntegrityError: null value in column "subpath" violates + # not-null constraint vulnerablecode_package = models.Package.objects.create( type=purl_to_dict.get("type"), namespace=purl_to_dict.get("namespace") or "", diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index 282228560..58731f852 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -504,3 +504,35 @@ ul.fixed_by_bullet li li li { .non-vuln .table th { border: solid 1px #dbdbdb; } + +/* Floating container to display the PURL on the Package details page as the user scrolls down. */ +.floating-purl { + position: sticky; + top: 0; + width: 100%; + z-index: 100; + margin-bottom: 0px; +} + +.floating-purl .table td, +.floating-purl .table tbody tr:last-child td, +.floating-purl .table th { + border: solid 1px #dbdbdb; + background-color: #ffffff; +} + +/* Emphasis for affected/fixed headings and related references. */ +.affected-fixed { + color: #ff0000; + font-weight: 700; +} + +/* Emphasis for not vulnerable. */ +.emphasis-not-vulnerable { + background-color: #e6ffe6; +} + +/* Emphasis for vulnerable. */ +.emphasis-vulnerable { + background-color: #ffe6e6; +} From 6a72c585c9a218009a352b1bb134b2a0a4a4525e Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 26 Sep 2023 19:02:56 -0700 Subject: [PATCH 38/53] Update Package details UI and Package API #1228 Reference: https://github.com/nexB/vulnerablecode/issues/1228 Signed-off-by: John M. Horan --- vulnerabilities/api.py | 140 ++---------------- vulnerabilities/models.py | 42 ++++-- .../templates/package_details.html | 70 ++++----- vulnerablecode/settings.py | 1 - vulnerablecode/static/css/custom.css | 5 +- 5 files changed, 77 insertions(+), 181 deletions(-) diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index a774ff0b6..6cfc37966 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -51,84 +51,26 @@ class MinimalPackageSerializer(serializers.HyperlinkedModelSerializer): affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities") def get_affected_vulnerabilities(self, package): - # HOT: 2023-09-13 Wednesday 13:48:01. 'parent_affected_vulnerabilities' identifies (1) the vulnerabilities (if any) that affect the given package that itself fixes one of the target purl's vulnerabilities, and (2) - # Ex: - # package.purl = pkg:pypi/redis@4.3.6 - # parent_affected_vulnerabilities = [{'vulnerability': , 'fixed_by_purl': PackageURL(type='pypi', namespace=None, name='redis', version='5.0.0b1', qualifiers={}, subpath=None), 'fixed_by_purl_vulnerabilities': []}] - parent_affected_vulnerabilities = package.fixed_package_details.get("vulnerabilities", None) affected_vulnerabilities = [] - print("\npackage.purl = {}".format(package.purl)) - print("\nparent_affected_vulnerabilities = {}\n".format(parent_affected_vulnerabilities)) - # print( - # "\nparent_affected_vulnerabilities.get('vulnerability', None) = {}\n".format( - # parent_affected_vulnerabilities.get("vulnerability", None) - # ) - # ) - # print( - # "\nparent_affected_vulnerabilities.get('fixed_by_purl', None) = {}\n".format( - # parent_affected_vulnerabilities.get("fixed_by_purl", None) - # ) - # ) - # print( - # "\nparent_affected_vulnerabilities.get('fixed_by_purl_vulnerabilities', None) = {}\n".format( - # parent_affected_vulnerabilities.get("fixed_by_purl_vulnerabilities", None) - # ) - # ) - if parent_affected_vulnerabilities: for vuln in parent_affected_vulnerabilities: affected_vulnerability = {} - print( - '\nvuln.get("vulnerability", None) = {}'.format(vuln.get("vulnerability", None)) - ) affected_vulnerability["vulnerability"] = vuln.get( "vulnerability", None ).vulnerability_id - print( - '\nvuln.get("fixed_by_purl", None) = {}'.format(vuln.get("fixed_by_purl", None)) - ) - affected_vulnerability["fixed_by_purl"] = vuln.get( - "fixed_by_purl", None - ).to_string() - - print( - '\nvuln.get("fixed_by_purl_vulnerabilities", None) = {}'.format( - vuln.get("fixed_by_purl_vulnerabilities", None) - ) - ) - affected_vulnerability["fixed_by_purl_vulnerabilities"] = [ - fixed_by_purl_vulnerability.vulnerability_id - for fixed_by_purl_vulnerability in vuln.get( - "fixed_by_purl_vulnerabilities", None - ) - ] - affected_vulnerabilities.append(affected_vulnerability) - print("affected_vulnerability = {}".format(affected_vulnerability)) - print("affected_vulnerabilities = {}".format(affected_vulnerabilities)) - return affected_vulnerabilities - # ZAP: 2023-09-12 Tuesday 19:12:22. Not what we want but this does return the purl, which we can use to find all of its own vulns (if any). - # return package.purl - - # def get_affected_vulnerabilities_saved(self, package) -> dict: - # """ - # Return a mapping of vulnerabilities that affects the given `package`. - # """ - # # return self.get_vulnerabilities_for_a_package(package=package, fix=False) - # return self.get_vulnerabilities_for_a_package(package=self, fix=False) - purl = serializers.CharField(source="package_url") class Meta: model = Package - fields = ["url", "purl", "is_vulnerable", "affected_by_vulnerabilities"] + fields = ["url", "purl", "affected_by_vulnerabilities"] class MinimalVulnerabilitySerializer(serializers.HyperlinkedModelSerializer): @@ -169,41 +111,9 @@ def to_representation(self, instance): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") - # ZAP: 2023-09-14 Thursday 11:34:26. Add closest fixed package from the Package UI here. - - closest_fixed_package = serializers.SerializerMethodField("get_closest_fixed_package") - - def get_closest_fixed_package(self, instance): - return "Coming soon!" - - # test_data = super().to_representation(instance) - # vulnerabilities = [vuln["vulnerability"] for vuln in test_data["vulnerabilities"]] - # test_data["vulnerabilities"] = vulnerabilities - - # return test_data - - # # ZAP: From the PackageSerializer class: - # # next_non_vulnerable_version = serializers.SerializerMethodField("get_closest_non_vulnerable") - - # def get_closest_non_vulnerable(self, package): - # closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) - # if closest_non_vulnerable: - # return closest_non_vulnerable.version - # else: - # return None - class Meta: model = Vulnerability - # fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] - fields = [ - "url", - "vulnerability_id", - "summary", - "references", - "fixed_packages", - "closest_fixed_package", - "aliases", - ] + fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer): @@ -216,34 +126,8 @@ class VulnerabilitySerializer(serializers.HyperlinkedModelSerializer): references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") aliases = AliasSerializer(many=True, source="alias") - # ALERT: 2023-09-14 Thursday 15:16:05. For some reason, favorite_color is not appearing in the output. - # favorite_color = serializers.SerializerMethodField("get_favorite_color") - - # # def get_closest_non_vulnerable(self, package): - # def get_favorite_color(self, package): - # # closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) - # # if closest_non_vulnerable: - # # return closest_non_vulnerable.version - # # else: - # # return None - # return "red!" - - favorite_color = MinimalPackageSerializer( - many=True, source="filtered_fixed_packages", read_only=True - ) - class Meta: model = Vulnerability - # fields = [ - # "url", - # "vulnerability_id", - # "summary", - # "aliases", - # "fixed_packages", - # "affected_packages", - # "references", - # ] - fields = [ "url", "vulnerability_id", @@ -252,7 +136,6 @@ class Meta: "fixed_packages", "affected_packages", "references", - "favorite_color", ] @@ -261,12 +144,12 @@ class PackageSerializer(serializers.HyperlinkedModelSerializer): Lookup software package using Package URLs """ - next_non_vulnerable_version = serializers.SerializerMethodField("get_closest_non_vulnerable") + next_non_vulnerable_version = serializers.SerializerMethodField("get_next_non_vulnerable") - def get_closest_non_vulnerable(self, package): - closest_non_vulnerable = package.fixed_package_details.get("closest_non_vulnerable", None) - if closest_non_vulnerable: - return closest_non_vulnerable.version + def get_next_non_vulnerable(self, package): + next_non_vulnerable = package.fixed_package_details.get("next_non_vulnerable", None) + if next_non_vulnerable: + return next_non_vulnerable.version else: return None @@ -287,7 +170,7 @@ def get_latest_non_vulnerable(self, package): def get_fixed_packages(self, package): """ - Return a queryset of all packages that fixes a vulnerability with + Return a queryset of all packages that fix a vulnerability with same type, namespace, name, subpath and qualifiers of the `package` """ return Package.objects.filter( @@ -302,7 +185,7 @@ def get_fixed_packages(self, package): def get_vulnerabilities_for_a_package(self, package, fix) -> dict: """ Return a mapping of vulnerabilities data related to the given `package`. - Return vulnerabilities that affects the `package` if given `fix` flag is False, + Return vulnerabilities that affect the `package` if given `fix` flag is False, otherwise return vulnerabilities fixed by the `package`. """ fixed_packages = self.get_fixed_packages(package=package) @@ -314,7 +197,6 @@ def get_vulnerabilities_for_a_package(self, package, fix) -> dict: to_attr="filtered_fixed_packages", ) ) - # so this calls VulnSerializerRefsAndSummary return VulnSerializerRefsAndSummary( instance=qs, many=True, @@ -344,10 +226,10 @@ class Meta: "version", "qualifiers", "subpath", - "affected_by_vulnerabilities", - "fixing_vulnerabilities", "next_non_vulnerable_version", "latest_non_vulnerable_version", + "affected_by_vulnerabilities", + "fixing_vulnerabilities", ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 75b834c71..3d97a09c4 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -687,14 +687,14 @@ def current_version(self): @property def fixed_package_details(self): """ - Return a mapping of vulnerabilities that affect this package and the closest and + Return a mapping of vulnerabilities that affect this package and the next and latest non-vulnerable versions. """ package_details = {} package_details["purl"] = PackageURL.from_string(self.purl) - closest_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() - package_details["closest_non_vulnerable"] = closest_non_vulnerable + next_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() + package_details["next_non_vulnerable"] = next_non_vulnerable package_details["latest_non_vulnerable"] = latest_non_vulnerable package_details["vulnerabilities"] = self.get_affecting_vulnerabilities() @@ -703,7 +703,7 @@ def fixed_package_details(self): def get_non_vulnerable_versions(self): """ - Return a tuple of the closest and latest non-vulnerable versions as PackageURLs. Return a tuple of + Return a tuple of the next and latest non-vulnerable versions as PackageURLs. Return a tuple of (None, None) if there is no non-vulnerable version. """ package_versions = self.get_fixed_by_package_versions(fix=False) @@ -720,13 +720,13 @@ def get_non_vulnerable_versions(self): if later_non_vulnerable_versions: sorted_versions = self.sort_by_version(later_non_vulnerable_versions) - closest_non_vulnerable_version = sorted_versions[0] + next_non_vulnerable_version = sorted_versions[0] latest_non_vulnerable_version = sorted_versions[-1] - closest_non_vulnerable = PackageURL.from_string(closest_non_vulnerable_version.purl) + next_non_vulnerable = PackageURL.from_string(next_non_vulnerable_version.purl) latest_non_vulnerable = PackageURL.from_string(latest_non_vulnerable_version.purl) - return closest_non_vulnerable, latest_non_vulnerable + return next_non_vulnerable, latest_non_vulnerable return None, None @@ -760,23 +760,33 @@ def get_affecting_vulnerabilities(self): if fixed_version > self.current_version: later_fixed_packages.append(fixed_pkg) - closest_fixed_package = None - closest_fixed_package_vulns = [] + next_fixed_package = None + next_fixed_package_vulns = [] + sort_fixed_by_packages_by_version = [] if later_fixed_packages: sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) - closest_fixed_package = sort_fixed_by_packages_by_version[0] - fixed_by_purl = PackageURL.from_string(closest_fixed_package.purl) - closest_fixed_package_vulns = list(closest_fixed_package.affected_by) + + fixed_by_pkgs = [] for vuln_details in package_details_vulns: if vuln_details["vulnerability"] != vuln: continue - vuln_details["fixed_by_purl"] = None + vuln_details["fixed_by_purl"] = [] vuln_details["fixed_by_purl_vulnerabilities"] = [] - if closest_fixed_package: - vuln_details["fixed_by_purl"] = fixed_by_purl - vuln_details["fixed_by_purl_vulnerabilities"] = closest_fixed_package_vulns + + for fixed_by_pkg in sort_fixed_by_packages_by_version: + fixed_by_package_details = {} + fixed_by_purl = PackageURL.from_string(fixed_by_pkg.purl) + next_fixed_package_vulns = list(fixed_by_pkg.affected_by) + + fixed_by_package_details["fixed_by_purl"] = fixed_by_purl + fixed_by_package_details[ + "fixed_by_purl_vulnerabilities" + ] = next_fixed_package_vulns + fixed_by_pkgs.append(fixed_by_package_details) + + vuln_details["fixed_by_package_details"] = fixed_by_pkgs return package_details_vulns diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 246b3733e..c70adaee0 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -50,8 +50,8 @@ Next non-vulnerable version - + @@ -116,45 +116,49 @@ {% if key == "vulnerabilities" %} {% for vuln in value %} {% if vuln.vulnerability.vulnerability_id == vulnerability.vulnerability_id %} - {% if vuln.fixed_by_purl is None %} - There are no reported fixed versions. + {% if vuln.fixed_by_package_details is None %} + There are no reported fixed by versions. {% else %} - {% if vuln.fixed_by_purl_vulnerabilities|length == 0 %} - {{ vuln.fixed_by_purl.version }} -
    - Affected by 0 other vulnerabilities. - {% else %} - {{ vuln.fixed_by_purl.version }} - {% if vuln.fixed_by_purl_vulnerabilities|length != 1 %} -
    - Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + {% for fixed_pkg in vuln.fixed_by_package_details %} +
    + {% if fixed_pkg.fixed_by_purl_vulnerabilities|length == 0 %} + {{ fixed_pkg.fixed_by_purl.version }} +
    + Affected by 0 other vulnerabilities. {% else %} -
    - Affected by {{ vuln.fixed_by_purl_vulnerabilities|length }} other vulnerability. - {% endif %} + {{ fixed_pkg.fixed_by_purl.version }} + {% if fixed_pkg.fixed_by_purl_vulnerabilities|length != 1 %} +
    + Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other vulnerabilities. + {% else %} +
    + Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other vulnerability. + {% endif %} -
    -
    - -
    -
    - There are no known affected packages. + There are no known affected packages.
    - See Affected packages tab for more + See Affected packages tab for more
    - {% if fixed_package_details.closest_non_vulnerable.version %} - {{ fixed_package_details.closest_non_vulnerable.version }} + {% if fixed_package_details.next_non_vulnerable.version %} + {{ fixed_package_details.next_non_vulnerable.version }} {% else %} None. {% endif %} @@ -85,7 +85,7 @@
    Vulnerability SummaryFixed byFixed by