Skip to content

Commit

Permalink
Merge pull request #1612 from aboutcode-org/new-models
Browse files Browse the repository at this point in the history
Segregate PackageRelatedVulnerability model to new models
  • Loading branch information
TG1999 authored Oct 22, 2024
2 parents 9910bef + 5081691 commit feab9fd
Show file tree
Hide file tree
Showing 19 changed files with 420 additions and 252 deletions.
7 changes: 0 additions & 7 deletions vulnerabilities/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

from vulnerabilities.models import ApiUser
from vulnerabilities.models import Package
from vulnerabilities.models import PackageRelatedVulnerability
from vulnerabilities.models import Vulnerability
from vulnerabilities.models import VulnerabilityReference
from vulnerabilities.models import VulnerabilitySeverity
Expand All @@ -35,12 +34,6 @@ class PackageAdmin(admin.ModelAdmin):
search_fields = ["name"]


@admin.register(PackageRelatedVulnerability)
class PackageRelatedVulnerabilityAdmin(admin.ModelAdmin):
list_filter = ("package__type", "package__namespace")
search_fields = ["vulnerability__vulnerability_id", "package__name"]


@admin.register(VulnerabilitySeverity)
class VulnerabilitySeverityAdmin(admin.ModelAdmin):
pass
Expand Down
22 changes: 12 additions & 10 deletions vulnerabilities/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ def get_fixed_packages(self, package):
type=package.type,
qualifiers=package.qualifiers,
subpath=package.subpath,
packagerelatedvulnerability__fix=True,
fixingpackagerelatedvulnerability__isnull=False,
)
.with_is_vulnerable()
.distinct()
Expand All @@ -300,10 +300,13 @@ def get_vulnerabilities_for_a_package(self, package, fix) -> dict:
otherwise return vulnerabilities fixed by the `package`.
"""
fixed_packages = self.get_fixed_packages(package=package)
qs = package.vulnerabilities.filter(packagerelatedvulnerability__fix=fix)
if fix:
qs = package.affected_by_vulnerabilities.all()
else:
qs = package.fixing_vulnerabilities.all()
qs = qs.prefetch_related(
Prefetch(
"packages",
"fixed_by_packages",
queryset=fixed_packages,
to_attr="filtered_fixed_packages",
)
Expand Down Expand Up @@ -372,7 +375,6 @@ class Meta:
"qualifiers",
"subpath",
"purl",
"packagerelatedvulnerability__fix",
]

def filter_purl(self, queryset, name, value):
Expand Down Expand Up @@ -590,7 +592,11 @@ def get_fixed_packages_qs(self):
Filter the packages that fixes a vulnerability
on fields like name, namespace and type.
"""
return self.get_packages_qs().filter(packagerelatedvulnerability__fix=True)
return (
self.get_packages_qs()
.filter(fixingpackagerelatedvulnerability__isnull=False)
.with_is_vulnerable()
)

def get_packages_qs(self):
"""
Expand All @@ -613,13 +619,9 @@ def get_queryset(self):
super()
.get_queryset()
.prefetch_related(
Prefetch(
"packages",
queryset=self.get_packages_qs(),
),
"weaknesses",
Prefetch(
"packages",
"fixed_by_packages",
queryset=self.get_fixed_packages_qs(),
to_attr="filtered_fixed_packages",
),
Expand Down
2 changes: 0 additions & 2 deletions vulnerabilities/api_extension.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,6 @@ class Meta:
"qualifiers",
"subpath",
"purl",
# this hurts
"packagerelatedvulnerability__fix",
]

def filter_purl(self, queryset, name, value):
Expand Down
9 changes: 4 additions & 5 deletions vulnerabilities/import_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@
from vulnerabilities.improver import Inference
from vulnerabilities.improvers.default import DefaultImporter
from vulnerabilities.models import Advisory
from vulnerabilities.models import AffectedByPackageRelatedVulnerability
from vulnerabilities.models import Alias
from vulnerabilities.models import FixingPackageRelatedVulnerability
from vulnerabilities.models import Package
from vulnerabilities.models import PackageRelatedVulnerability
from vulnerabilities.models import Vulnerability
from vulnerabilities.models import VulnerabilityChangeLog
from vulnerabilities.models import VulnerabilityReference
Expand Down Expand Up @@ -211,22 +212,20 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver

for affected_purl in inference.affected_purls or []:
vulnerable_package, _ = Package.objects.get_or_create_from_purl(purl=affected_purl)
PackageRelatedVulnerability(
AffectedByPackageRelatedVulnerability(
vulnerability=vulnerability,
package=vulnerable_package,
created_by=improver_name,
confidence=inference.confidence,
fix=False,
).update_or_create(advisory=advisory)

if inference.fixed_purl:
fixed_package, _ = Package.objects.get_or_create_from_purl(purl=inference.fixed_purl)
PackageRelatedVulnerability(
FixingPackageRelatedVulnerability(
vulnerability=vulnerability,
package=fixed_package,
created_by=improver_name,
confidence=inference.confidence,
fix=True,
).update_or_create(advisory=advisory)

if inference.weaknesses and vulnerability:
Expand Down
9 changes: 4 additions & 5 deletions vulnerabilities/improve_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@
from vulnerabilities.importers import IMPORTERS_REGISTRY
from vulnerabilities.improver import Inference
from vulnerabilities.models import Advisory
from vulnerabilities.models import AffectedByPackageRelatedVulnerability
from vulnerabilities.models import Alias
from vulnerabilities.models import FixingPackageRelatedVulnerability
from vulnerabilities.models import Package
from vulnerabilities.models import PackageRelatedVulnerability
from vulnerabilities.models import Vulnerability
from vulnerabilities.models import VulnerabilityChangeLog
from vulnerabilities.models import VulnerabilityReference
Expand Down Expand Up @@ -135,12 +136,11 @@ def process_inferences(
vulnerable_package, created = Package.objects.get_or_create_from_purl(
purl=affected_purl
)
PackageRelatedVulnerability(
AffectedByPackageRelatedVulnerability(
vulnerability=vulnerability,
package=vulnerable_package,
created_by=improver_name,
confidence=inference.confidence,
fix=False,
).update_or_create(
advisory=advisory,
)
Expand All @@ -149,12 +149,11 @@ def process_inferences(
fixed_package, created = Package.objects.get_or_create_from_purl(
purl=inference.fixed_purl
)
PackageRelatedVulnerability(
FixingPackageRelatedVulnerability(
vulnerability=vulnerability,
package=fixed_package,
created_by=improver_name,
confidence=inference.confidence,
fix=True,
).update_or_create(
advisory=advisory,
)
Expand Down
14 changes: 9 additions & 5 deletions vulnerabilities/management/commands/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def export_data(self, base_path: Path):
}
package_vulnerabilities.append(package_data)

for vuln in pkg_version.vulnerabilities.all():
for vuln in pkg_version.vulnerabilities:
vcid = vuln.vulnerability_id
# do not write twice the same file
if vcid in seen_vcid:
Expand Down Expand Up @@ -158,10 +158,14 @@ def packages_by_type_ns_name():
qs = (
Package.objects.order_by("type", "namespace", "name", "version")
.prefetch_related(
"vulnerabilities",
"vulnerabilities__references",
"vulnerabilities__weaknesses",
"vulnerabilities__references__vulnerabilityseverity_set",
"affected_by_vulnerabilities",
"affected_by_vulnerabilities__references",
"affected_by_vulnerabilities__weaknesses",
"affected_by_vulnerabilities__references__vulnerabilityseverity_set",
"fixing_vulnerabilities",
"fixing_vulnerabilities__references",
"fixing_vulnerabilities__weaknesses",
"fixing_vulnerabilities__references__vulnerabilityseverity_set",
)
.paginated()
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,4 @@ class Migration(migrations.Migration):
max_length=100,
),
),
]
]
164 changes: 164 additions & 0 deletions vulnerabilities/migrations/0071_auto_20241007_1044.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
from django.db import migrations, models
import django.db.models.deletion
from django.core.validators import MaxValueValidator, MinValueValidator
from vulnerabilities.improver import MAX_CONFIDENCE

def split_packagerelatedvulnerability(apps, schema_editor):
PackageRelatedVulnerability = apps.get_model('vulnerabilities', 'PackageRelatedVulnerability')
FixingPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'FixingPackageRelatedVulnerability')
AffectedByPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'AffectedByPackageRelatedVulnerability')

for prv in PackageRelatedVulnerability.objects.all():
if prv.fix:
FixingPackageRelatedVulnerability.objects.create(
package=prv.package,
vulnerability=prv.vulnerability,
created_by=prv.created_by,
confidence=prv.confidence,
)
else:
AffectedByPackageRelatedVulnerability.objects.create(
package=prv.package,
vulnerability=prv.vulnerability,
created_by=prv.created_by,
confidence=prv.confidence,
)

def reverse_migration(apps, schema_editor):
FixingPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'FixingPackageRelatedVulnerability')
AffectedByPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'AffectedByPackageRelatedVulnerability')
PackageRelatedVulnerability = apps.get_model('vulnerabilities', 'PackageRelatedVulnerability')

for fpv in FixingPackageRelatedVulnerability.objects.all():
PackageRelatedVulnerability.objects.create(
package=fpv.package,
vulnerability=fpv.vulnerability,
created_by=fpv.created_by,
confidence=fpv.confidence,
fix=True,
)

for apv in AffectedByPackageRelatedVulnerability.objects.all():
PackageRelatedVulnerability.objects.create(
package=apv.package,
vulnerability=apv.vulnerability,
created_by=apv.created_by,
confidence=apv.confidence,
fix=False,
)

class Migration(migrations.Migration):

dependencies = [
("vulnerabilities", "0070_alter_advisory_created_by_and_more"),
]

operations = [
migrations.AlterField(
model_name="advisory",
name="created_by",
field=models.CharField(
help_text="Fully qualified name of the importer prefixed with themodule name importing the advisory. Eg:vulnerabilities.pipeline.nginx_importer.NginxImporterPipeline",
max_length=100,
),
),
migrations.CreateModel(
name="FixingPackageRelatedVulnerability",
fields=[
(
"id",
models.AutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
(
"created_by",
models.CharField(
blank=True,
help_text="Fully qualified name of the improver prefixed with the module name responsible for creating this relation. Eg: vulnerabilities.importers.nginx.NginxBasicImprover",
max_length=100,
),
),
(
"confidence",
models.PositiveIntegerField(
default=100,
help_text="Confidence score for this relation",
validators=[
django.core.validators.MinValueValidator(0),
django.core.validators.MaxValueValidator(100),
],
),
),
(
"package",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.package"
),
),
(
"vulnerability",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="vulnerabilities.vulnerability",
),
),
],
options={
"verbose_name_plural": "Fixing Package Related Vulnerabilities",
"ordering": ["package", "vulnerability"],
"abstract": False,
"unique_together": {("package", "vulnerability")},
},
),
migrations.CreateModel(
name="AffectedByPackageRelatedVulnerability",
fields=[
(
"id",
models.AutoField(
auto_created=True, primary_key=True, serialize=False, verbose_name="ID"
),
),
(
"created_by",
models.CharField(
blank=True,
help_text="Fully qualified name of the improver prefixed with the module name responsible for creating this relation. Eg: vulnerabilities.importers.nginx.NginxBasicImprover",
max_length=100,
),
),
(
"confidence",
models.PositiveIntegerField(
default=100,
help_text="Confidence score for this relation",
validators=[
django.core.validators.MinValueValidator(0),
django.core.validators.MaxValueValidator(100),
],
),
),
(
"package",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.package"
),
),
(
"vulnerability",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
to="vulnerabilities.vulnerability",
),
),
],
options={
"verbose_name_plural": "Affected By Package Related Vulnerabilities",
"ordering": ["package", "vulnerability"],
"abstract": False,
"unique_together": {("package", "vulnerability")},
},
),
migrations.RunPython(split_packagerelatedvulnerability, reverse_migration),
]
Loading

0 comments on commit feab9fd

Please sign in to comment.