From 56fa9f8a6cc0bf8eb1c04a1feb9ed929928edb2d Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Tue, 30 May 2023 11:29:45 +0800 Subject: [PATCH 01/10] feat: new sca impl. --- dongtai_common/models/asset_vul_v2.py | 64 +++++++++ dongtai_common/models/assetv2.py | 125 ++++++++++++++++++ dongtai_web/dongtai_sca/common/dataclass.py | 85 ++++++++++++ dongtai_web/dongtai_sca/urls.py | 24 +++- dongtai_web/dongtai_sca/views/newpackage.py | 55 ++++++++ .../dongtai_sca/views/newpackagedetail.py | 52 ++++++++ .../dongtai_sca/views/newpackageprojects.py | 58 ++++++++ .../views/newpackageprojectversions.py | 56 ++++++++ .../dongtai_sca/views/newpackagesummary.py | 82 ++++++++++++ .../dongtai_sca/views/newpackagevuldetail.py | 52 ++++++++ .../dongtai_sca/views/newpackagevuls.py | 77 +++++++++++ dongtai_web/utils.py | 15 +++ 12 files changed, 744 insertions(+), 1 deletion(-) create mode 100644 dongtai_common/models/asset_vul_v2.py create mode 100644 dongtai_common/models/assetv2.py create mode 100644 dongtai_web/dongtai_sca/common/dataclass.py create mode 100644 dongtai_web/dongtai_sca/views/newpackage.py create mode 100644 dongtai_web/dongtai_sca/views/newpackagedetail.py create mode 100644 dongtai_web/dongtai_sca/views/newpackageprojects.py create mode 100644 dongtai_web/dongtai_sca/views/newpackageprojectversions.py create mode 100644 dongtai_web/dongtai_sca/views/newpackagesummary.py create mode 100644 dongtai_web/dongtai_sca/views/newpackagevuldetail.py create mode 100644 dongtai_web/dongtai_sca/views/newpackagevuls.py diff --git a/dongtai_common/models/asset_vul_v2.py b/dongtai_common/models/asset_vul_v2.py new file mode 100644 index 000000000..e071e6ec8 --- /dev/null +++ b/dongtai_common/models/asset_vul_v2.py @@ -0,0 +1,64 @@ +import uuid +from dongtai_common.models.agent import IastAgent +from django.core.cache import cache +from django_elasticsearch_dsl.search import Search +from dongtai_conf.settings import ASSET_VUL_INDEX +from django_elasticsearch_dsl import Document, fields +from django_elasticsearch_dsl.registries import registry +from dongtai_common.models.assetv2 import AssetV2Global +from django.db import models +from dongtai_common.utils.settings import get_managed +from dongtai_common.models.vulnerablity import IastVulnerabilityStatus +from dongtai_common.models.vul_level import IastVulLevel + + +class IastAssetVulV2(models.Model): + vul_id = models.CharField(max_length=255, blank=True, null=True) + vul_name = models.CharField(max_length=255, blank=True, null=True) + vul_detail = models.TextField(blank=True, null=True) + # 漏洞类型等级 + level = models.ForeignKey(IastVulLevel, + models.DO_NOTHING, + blank=True, + null=True) + vul_type = models.CharField(max_length=255, blank=True, null=True) + package_language = models.CharField(max_length=10, blank=True, null=True) + published_time = models.IntegerField(blank=True, null=True) + change_time = models.IntegerField(blank=True, null=True) + update_time = models.IntegerField(blank=True, null=True) + create_time = models.IntegerField(blank=True, null=True) + references = models.JSONField(blank=True, null=True, default=list) + asset = models.ManyToManyField('IastVulAssetRelationV2') + + class Meta: + managed = True + db_table = 'iast_asset_vul_v2' + + +class IastVulAssetRelationV2(models.Model): + asset_vul = models.ForeignKey(IastAssetVulV2, + on_delete=models.DO_NOTHING, + db_constraint=False, + db_column='vul_id') + assee = models.ForeignKey(AssetV2Global, + on_delete=models.DO_NOTHING, + db_constraint=False, + db_column='aql') + create_time = models.IntegerField(blank=True, null=True) + + class Meta: + managed = get_managed() + db_table = 'iast_asset_vul_v2_relation' + + +class IastPackageGAInfo(models.Model): + package_name = models.ForeignKey(AssetV2Global, + on_delete=models.DO_NOTHING, + db_constraint=False, + db_column='package_name') + affected_versions = models.JSONField(blank=True, null=True, default=list) + unaffected_versions = models.JSONField(blank=True, null=True, default=list) + + class Meta: + managed = get_managed() + db_table = 'iast_asset_v2_ga_info' diff --git a/dongtai_common/models/assetv2.py b/dongtai_common/models/assetv2.py new file mode 100644 index 000000000..e7a0e17ee --- /dev/null +++ b/dongtai_common/models/assetv2.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +# author:owefsad +# datetime:2020/8/20 15:10 +# software: PyCharm +# project: dongtai-models + +import uuid +from django.core.cache import cache +from django_elasticsearch_dsl.search import Search +from dongtai_conf.settings import ASSET_INDEX +from django_elasticsearch_dsl import Document, fields +from django.db.models.fields.related import ForeignKey +from dongtai_web.utils import get_model_field +from django_elasticsearch_dsl.registries import registry +from django_elasticsearch_dsl import Document +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from dongtai_common.models import User +from dongtai_common.models.agent import IastAgent +from dongtai_common.models.project import IastProject +from dongtai_common.models.project_version import IastProjectVersion +from dongtai_common.models.vul_level import IastVulLevel +from dongtai_common.utils.settings import get_managed +from dongtai_common.models.department import Department +from dongtai_common.models.talent import Talent + + +class AssetV2(models.Model): + id = models.BigAutoField(primary_key=True) + package_name = models.CharField(max_length=255, blank=True, null=True) + aql = models.ForeignKey('AssetV2Global', + to_field='aql', + default='', + on_delete=models.DO_NOTHING) + package_path = models.CharField(max_length=255, blank=True, null=True) + signature_algorithm = models.CharField(max_length=255, + blank=True, + null=True) + signature_value = models.CharField(max_length=255, blank=True, null=True) + dt = models.IntegerField(blank=True, null=True) + is_reconized = models.IntegerField(blank=True, null=True) + version = models.CharField(max_length=255, blank=True, null=True) + level = models.ForeignKey(IastVulLevel, + models.DO_NOTHING, + blank=True, + null=True, + default=4) + project = models.ForeignKey(IastProject, + on_delete=models.CASCADE, + blank=True, + null=False, + default=-1) + project_version = models.ForeignKey(IastProjectVersion, + on_delete=models.CASCADE, + blank=True, + null=False, + default=-1) + language = models.CharField(max_length=32, + blank=True, + null=False, + default='') + + # 部门id + department = models.ForeignKey(Department, + models.DO_NOTHING, + blank=True, + null=True, + default=-1) + + class Meta: + managed = get_managed() + db_table = 'iast_asset_v2' + + +class AssetV2Global(models.Model): + id = models.BigAutoField(primary_key=True) + aql = models.CharField(max_length=255, blank=True, null=True) + package_name = models.CharField(max_length=255, blank=True, null=True) + signature_algorithm = models.CharField(max_length=255, + blank=True, + null=True) + signature_value = models.CharField(max_length=255, blank=True, null=True) + version = models.CharField(max_length=255, blank=True, null=True) + level = models.ForeignKey(IastVulLevel, + models.DO_NOTHING, + blank=True, + null=True, + default=4) + vul_count = models.IntegerField(blank=True, null=True) + vul_critical_count = models.IntegerField(default=0, blank=True, null=False) + vul_high_count = models.IntegerField(default=0, blank=True, null=False) + vul_medium_count = models.IntegerField(default=0, blank=True, null=False) + vul_low_count = models.IntegerField(default=0, blank=True, null=False) + vul_info_count = models.IntegerField(default=0, blank=True, null=False) + language = models.CharField(max_length=32, + blank=True, + null=False, + default='') + language_id = models.IntegerField(default=1, blank=True, null=False) + + safe_version_list = models.JSONField(blank=True, null=True, default=list) + license_list = models.JSONField(blank=True, null=True, default=list) + + class Meta: + managed = get_managed() + db_table = 'iast_asset_v2_summary' + + +class IastAssetLicense(models.Model): + license_id = models.IntegerField(blank=True, null=True) + license = models.CharField(max_length=64, + blank=True, + null=False, + default='') + asset = models.ForeignKey(AssetV2Global, + on_delete=models.DO_NOTHING, + db_constraint=False, + db_column='asset_id') + create_time = models.IntegerField(blank=True, null=True) + + class Meta: + managed = get_managed() + db_table = 'iast_asset_v2_license' diff --git a/dongtai_web/dongtai_sca/common/dataclass.py b/dongtai_web/dongtai_sca/common/dataclass.py new file mode 100644 index 000000000..d5efc101f --- /dev/null +++ b/dongtai_web/dongtai_sca/common/dataclass.py @@ -0,0 +1,85 @@ +from typing import List +from typing import Any +from typing import Optional +from dataclasses import dataclass, field +from dataclasses_json import dataclass_json, config +from datetime import datetime + +@dataclass_json +@dataclass +class Reference: + type: str = "" + url: str = "" + + +@dataclass_json +@dataclass +class Root: + status: int + msg: str + data: Data + + +@dataclass_json +@dataclass +class VulCodes: + CVE: List[str] = [] + GHSA: List[str] = [] + + +@dataclass_json +@dataclass +class VulInfo: + vul_id: str = "" + cvss_v3: str = "" + cwe: List[str] = [] + title: str = "" + description: str = "" + references: List[Reference] = [] + severity: str = [] + published_time: Optional[datetime] = field( + default=None, + metadata=config(decoder=lambda x: datetime.fromisoformat(x) + if x is not None else None, + encoder=lambda x: datetime.isoformat(x) + if x is not None else None)) + create_time: datetime = field(default=datetime.now(), + metadata=config( + decoder=datetime.fromisoformat, + encoder=datetime.isoformat)) + update_time: datetime = field(default=datetime.now(), + metadata=config( + decoder=datetime.fromisoformat, + encoder=datetime.isoformat)) + change_time: datetime = field(default=datetime.now(), + metadata=config( + decoder=datetime.fromisoformat, + encoder=datetime.isoformat)) + + +@dataclass_json +@dataclass +class Vul: + vul_info: VulInfo + vul_codes: VulCodes + affected_versions: List[str] = [] + + +@dataclass_json +@dataclass +class Data: + vuls: List[Vul] = [] + affected_versions: List[str] = [] + unaffected_versions: List[str] = [] + +@dataclass_json +@dataclass +class PackageInfo: + ecosystem: str + language: str + name: str + version: str + hash: str + license: List[str] + version_publish_time: str + diff --git a/dongtai_web/dongtai_sca/urls.py b/dongtai_web/dongtai_sca/urls.py index ff4c32895..fc569a9ff 100644 --- a/dongtai_web/dongtai_sca/urls.py +++ b/dongtai_web/dongtai_sca/urls.py @@ -2,6 +2,13 @@ from dongtai_web.dongtai_sca.views.asset_projects import AssetProjects from dongtai_web.dongtai_sca.views.package import PackageList, AssetAggrDetailAssetIds +from dongtai_web.dongtai_sca.views.newpackage import PackageList +from dongtai_web.dongtai_sca.views.newpackagedetail import PackageDetail +from dongtai_web.dongtai_sca.views.newpackagevuldetail import PackageVulDetail +from dongtai_web.dongtai_sca.views.newpackagesummary import NewPackageSummary +from dongtai_web.dongtai_sca.views.newpackagevuls import NewPackageVuls +from dongtai_web.dongtai_sca.views.newpackageprojects import NewPackageRelationProject +from dongtai_web.dongtai_sca.views.newpackageprojectversions import NewPackageRelationProjectVersion from dongtai_web.dongtai_sca.views.package_vul import OnePackageVulList, AssetPackageVulList, AssetPackageVulDetail from rest_framework import routers @@ -16,4 +23,19 @@ path('asset_ids/', AssetAggrDetailAssetIds.as_view()), ] -urlpatterns = [path('sca/v1/', include(urlpatterns), name='ScaAPI'), ] +v2_urlpatterns = [ + path('package/', PackageList.as_view()), + path('package//', + PackageDetail.as_view()), + path('package_summary/', NewPackageSummary.as_view()), + path('package_vuls/', NewPackageVuls.as_view()), + path('package_vul_detail/', PackageVulDetail.as_view()), + path('package_realtion_project/', NewPackageRelationProject.as_view()), + path('package_realtion_project_version/', + NewPackageRelationProjectVersion.as_view()), +] + +urlpatterns = [ + path('sca/v1/', include(urlpatterns), name='ScaAPI'), + path('sca/v2/', include(v2_urlpatterns), name='ScaAPI'), +] diff --git a/dongtai_web/dongtai_sca/views/newpackage.py b/dongtai_web/dongtai_sca/views/newpackage.py new file mode 100644 index 000000000..8eaf29053 --- /dev/null +++ b/dongtai_web/dongtai_sca/views/newpackage.py @@ -0,0 +1,55 @@ +import logging + +from dongtai_common.models import User +from dongtai_web.dongtai_sca.models import Package +from django.http import JsonResponse +from rest_framework import views +from django.core.paginator import Paginator +from django.forms.models import model_to_dict +from dongtai_common.endpoint import R, UserEndPoint +from django.utils.translation import gettext_lazy as _ +from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer +from rest_framework import serializers + +from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id +from dongtai_common.models.assetv2 import AssetV2, AssetV2Global + +logger = logging.getLogger(__name__) + + +class PackageListArgsSerializer(serializers.Serializer): + page_size = serializers.IntegerField(default=20, + help_text=_('Number per page')) + page = serializers.IntegerField(default=1, help_text=_('Page index')) + language_ids = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('language'))) + license_ids = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('license'))) + level_ids = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('level'))) + project_id = serializers.IntegerField(default=1, help_text=_('Page index')) + project_version_id = serializers.IntegerField(default=1, + help_text=_('Page index')) + keyword = serializers.CharField(help_text=_("search_keyword")) + order_field = serializers.ChoiceField(['vul_count', 'level'], + default='level') + order = serializers.ChoiceField(['desc', 'asc'], default='desc') + + +class PackeageScaAssetSerializer(serializers.ModelSerializer): + + class Meta: + model = AssetV2Global + fields = '__all__' + + +_NewResponseSerializer = get_response_serializer( + PackeageScaAssetSerializer(many=True)) + + +class PackageList(UserEndPoint): + + @extend_schema_with_envcheck_v2(request=PackageListArgsSerializer, + responses={200: _NewResponseSerializer}) + def post(self, request): + return JsonResponse({}) diff --git a/dongtai_web/dongtai_sca/views/newpackagedetail.py b/dongtai_web/dongtai_sca/views/newpackagedetail.py new file mode 100644 index 000000000..ff5766c0d --- /dev/null +++ b/dongtai_web/dongtai_sca/views/newpackagedetail.py @@ -0,0 +1,52 @@ +import logging + +from dongtai_common.models import User +from dongtai_web.dongtai_sca.models import Package +from django.http import JsonResponse +from rest_framework import views +from django.core.paginator import Paginator +from django.forms.models import model_to_dict +from dongtai_common.endpoint import R, UserEndPoint +from django.utils.translation import gettext_lazy as _ +from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer +from rest_framework import serializers + +from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id +from dongtai_common.models.assetv2 import AssetV2, AssetV2Global + +logger = logging.getLogger(__name__) + + +class PackageListArgsSerializer(serializers.Serializer): + page_size = serializers.IntegerField(default=20, + help_text=_('Number per page')) + page = serializers.IntegerField(default=1, help_text=_('Page index')) + languages = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('language'))) + licenses = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('license'))) + levels = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('level'))) + project_id = serializers.IntegerField(default=1, help_text=_('Page index')) + project_version_id = serializers.IntegerField(default=1, + help_text=_('Page index')) + keyword = serializers.CharField(help_text=_("search_keyword")) + order_field = serializers.CharField(help_text=_("order_field")) + order = serializers.CharField(help_text=_("order")) + + +class PackeageScaAssetSerializer(serializers.ModelSerializer): + + class Meta: + model = AssetV2Global + fields = '__all__' + + +_NewResponseSerializer = get_response_serializer(PackeageScaAssetSerializer()) + + +class PackageDetail(UserEndPoint): + + @extend_schema_with_envcheck_v2(responses={200: _NewResponseSerializer}) + def get(self, request, package_name, package_version): + return JsonResponse({}) diff --git a/dongtai_web/dongtai_sca/views/newpackageprojects.py b/dongtai_web/dongtai_sca/views/newpackageprojects.py new file mode 100644 index 000000000..bf579fbe5 --- /dev/null +++ b/dongtai_web/dongtai_sca/views/newpackageprojects.py @@ -0,0 +1,58 @@ +import logging + +from dongtai_common.models import User +from dongtai_web.dongtai_sca.models import Package +from django.http import JsonResponse +from rest_framework import views +from django.core.paginator import Paginator +from django.forms.models import model_to_dict +from dongtai_common.endpoint import R, UserEndPoint +from django.utils.translation import gettext_lazy as _ +from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer +from rest_framework import serializers + +from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id +from dongtai_common.models.assetv2 import AssetV2, AssetV2Global +from rest_framework_dataclasses.serializers import DataclassSerializer +from dataclasses import dataclass, field +from typing import List +from typing import Any +from dataclasses import dataclass +import json + +logger = logging.getLogger(__name__) + + +@dataclass +class RelationProject: + project_id: int + project_name: str + + +class RelationProjectArgsSerializer(serializers.Serializer): + page_size = serializers.IntegerField(default=20, + help_text=_('Number per page')) + page = serializers.IntegerField(default=1, help_text=_('Page index')) + package_name = serializers.CharField(help_text=_("order_field")) + package_version = serializers.CharField(help_text=_("order")) + project_id = serializers.IntegerField( + help_text=_("project with be the first")) + + +class RelationProjectSerializer(DataclassSerializer): + + class Meta: + dataclass = RelationProject + + +FullRelationProjectResponseSerializer = get_response_serializer( + RelationProjectSerializer(many=True)) + + +class NewPackageRelationProject(UserEndPoint): + + @extend_schema_with_envcheck_v2( + request=RelationProjectArgsSerializer, + responses={200: FullRelationProjectResponseSerializer}) + def post(self, request): + return JsonResponse({}) diff --git a/dongtai_web/dongtai_sca/views/newpackageprojectversions.py b/dongtai_web/dongtai_sca/views/newpackageprojectversions.py new file mode 100644 index 000000000..6ef2d8c29 --- /dev/null +++ b/dongtai_web/dongtai_sca/views/newpackageprojectversions.py @@ -0,0 +1,56 @@ +import logging + +from dongtai_common.models import User +from dongtai_web.dongtai_sca.models import Package +from django.http import JsonResponse +from rest_framework import views +from django.core.paginator import Paginator +from django.forms.models import model_to_dict +from dongtai_common.endpoint import R, UserEndPoint +from django.utils.translation import gettext_lazy as _ +from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer +from rest_framework import serializers + +from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id +from dongtai_common.models.assetv2 import AssetV2, AssetV2Global +from rest_framework_dataclasses.serializers import DataclassSerializer +from dataclasses import dataclass, field +from typing import List +from typing import Any +from dataclasses import dataclass +import json + +logger = logging.getLogger(__name__) + + +@dataclass +class RelationProjectVersion: + project_version_name: str + package_path: str + + +class RelationProjectVersionArgsSerializer(serializers.Serializer): + page_size = serializers.IntegerField(default=20, + help_text=_('Number per page')) + page = serializers.IntegerField(default=1, help_text=_('Page index')) + project_id = serializers.IntegerField( + help_text=_("project with be the first")) + + +class RelationProjectVersionSerializer(DataclassSerializer): + + class Meta: + dataclass = RelationProjectVersion + + +FullRelationProjectVersionResponseSerializer = get_response_serializer( + RelationProjectVersionSerializer(many=True)) + + +class NewPackageRelationProjectVersion(UserEndPoint): + + @extend_schema_with_envcheck_v2( + request=RelationProjectVersionArgsSerializer, + responses={200: FullRelationProjectVersionResponseSerializer}) + def post(self, request): + return JsonResponse({}) diff --git a/dongtai_web/dongtai_sca/views/newpackagesummary.py b/dongtai_web/dongtai_sca/views/newpackagesummary.py new file mode 100644 index 000000000..7f4b5e587 --- /dev/null +++ b/dongtai_web/dongtai_sca/views/newpackagesummary.py @@ -0,0 +1,82 @@ +import logging + +from dongtai_common.models import User +from dongtai_web.dongtai_sca.models import Package +from django.http import JsonResponse +from rest_framework import views +from django.core.paginator import Paginator +from django.forms.models import model_to_dict +from dongtai_common.endpoint import R, UserEndPoint +from django.utils.translation import gettext_lazy as _ +from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer +from rest_framework import serializers + +from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id +from dongtai_common.models.assetv2 import AssetV2, AssetV2Global +from rest_framework_dataclasses.serializers import DataclassSerializer +from dataclasses import dataclass, field +from typing import List +from typing import Any +from dataclasses import dataclass +import json + +logger = logging.getLogger(__name__) + + +@dataclass +class Language: + language: str + count: int + language_id: int + + +@dataclass +class Level: + level: str + count: int + level_id: int + + +@dataclass +class License: + license: str + count: int + license_id: int + + +@dataclass +class Data: + level: List[Level] + language: List[Language] + license: List[License] + + +class PackageSummaryArgsSerializer(serializers.Serializer): + language_ids = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('language'))) + license_ids = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('license'))) + level_ids = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('level'))) + project_id = serializers.IntegerField(default=1, help_text=_('Page index')) + project_version_id = serializers.IntegerField(default=1, + help_text=_('Page index')) + + +class PackeageScaSummarySerializer(DataclassSerializer): + + class Meta: + dataclass = Data + + +FullSummaryResponseSerializer = get_response_serializer( + PackeageScaSummarySerializer(many=True)) + + +class NewPackageSummary(UserEndPoint): + + @extend_schema_with_envcheck_v2( + request=PackageSummaryArgsSerializer, + responses={200: FullSummaryResponseSerializer}) + def post(self, request): + return JsonResponse({}) diff --git a/dongtai_web/dongtai_sca/views/newpackagevuldetail.py b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py new file mode 100644 index 000000000..eda51d7c8 --- /dev/null +++ b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py @@ -0,0 +1,52 @@ +import logging + +from dongtai_common.models import User +from dongtai_web.dongtai_sca.models import Package +from django.http import JsonResponse +from rest_framework import views +from django.core.paginator import Paginator +from django.forms.models import model_to_dict +from dongtai_common.endpoint import R, UserEndPoint +from django.utils.translation import gettext_lazy as _ +from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer +from rest_framework import serializers + +from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id +from dongtai_common.models.asset_vul_v2 import IastAssetVulV2 + +logger = logging.getLogger(__name__) + + +class PackageListArgsSerializer(serializers.Serializer): + page_size = serializers.IntegerField(default=20, + help_text=_('Number per page')) + page = serializers.IntegerField(default=1, help_text=_('Page index')) + language_ids = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('language'))) + license_ids = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('license'))) + level_ids = serializers.ListField( + child=serializers.IntegerField(default=1, help_text=_('level'))) + project_id = serializers.IntegerField(default=1, help_text=_('Page index')) + project_version_id = serializers.IntegerField(default=1, + help_text=_('Page index')) + keyword = serializers.CharField(help_text=_("search_keyword")) + order_field = serializers.CharField(help_text=_("order_field")) + order = serializers.CharField(help_text=_("order")) + + +class PackeageVulSerializer(serializers.ModelSerializer): + + class Meta: + model = IastAssetVulV2 + fields = '__all__' + + +_NewResponseSerializer = get_response_serializer(PackeageVulSerializer()) + + +class PackageVulDetail(UserEndPoint): + + @extend_schema_with_envcheck_v2(responses={200: _NewResponseSerializer}) + def get(self, request, vul_id): + return JsonResponse({}) diff --git a/dongtai_web/dongtai_sca/views/newpackagevuls.py b/dongtai_web/dongtai_sca/views/newpackagevuls.py new file mode 100644 index 000000000..60cd5ca07 --- /dev/null +++ b/dongtai_web/dongtai_sca/views/newpackagevuls.py @@ -0,0 +1,77 @@ +import logging + +from dongtai_common.models import User +from dongtai_web.dongtai_sca.models import Package +from django.http import JsonResponse +from rest_framework import views +from django.core.paginator import Paginator +from django.forms.models import model_to_dict +from dongtai_common.endpoint import R, UserEndPoint +from django.utils.translation import gettext_lazy as _ +from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer +from rest_framework import serializers + +from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id +from dongtai_common.models.asset_vul_v2 import IastAssetVulV2 +from rest_framework_dataclasses.serializers import DataclassSerializer +from dataclasses import dataclass, field +from typing import List +from typing import Any +from dataclasses import dataclass +import json + +logger = logging.getLogger(__name__) + + +#@dataclass +#class Language: +# language: str +# count: int +# +# +#@dataclass +#class Level: +# level: str +# count: int +# level_id: int +# +# +#@dataclass +#class License: +# license: str +# count: int +# +# +#@dataclass +#class Data: +# level: List[Level] +# language: List[Language] +# license: List[License] + + +class PackageVulsListArgsSerializer(serializers.Serializer): + page_size = serializers.IntegerField(default=20, + help_text=_('Number per page')) + page = serializers.IntegerField(default=1, help_text=_('Page index')) + package_name = serializers.CharField(help_text=_("pacakge name")) + package_version = serializers.CharField(help_text=_("package version")) + + +class PackeageVulsSerializer(serializers.ModelSerializer): + + class Meta: + model = IastAssetVulV2 + fields = '__all__' + + +NewPackageVulSResponseSerializer = get_response_serializer( + PackeageVulsSerializer(many=True)) + + +class NewPackageVuls(UserEndPoint): + + @extend_schema_with_envcheck_v2( + request=PackageVulsListArgsSerializer, + responses={200: NewPackageVulSResponseSerializer}) + def post(self, request): + return JsonResponse({}) diff --git a/dongtai_web/utils.py b/dongtai_web/utils.py index d7747918e..dcc72c85e 100644 --- a/dongtai_web/utils.py +++ b/dongtai_web/utils.py @@ -124,6 +124,21 @@ def myextend_schema(func): return myextend_schema +def extend_schema_with_envcheck_v2(*args, **kwargs): + + def myextend_schema(func): + import os + if os.getenv('environment', None) in ('TEST', 'DOC') or os.getenv( + 'DOC', None) == 'TRUE': + from drf_spectacular.utils import extend_schema, OpenApiParameter, OpenApiExample, OpenApiTypes + deco = extend_schema(*args, **kwargs) + funcw = deco(func) + return funcw + return func + + return myextend_schema + + def get_response_serializer(data_serializer=None, msg_list=None, status_msg_keypair=None): From 9cc2e7eaa757f1e0988458aed9b3718828f8c69d Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Tue, 30 May 2023 11:42:44 +0800 Subject: [PATCH 02/10] feat: new sca impl. --- dongtai_common/models/assetv2.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/dongtai_common/models/assetv2.py b/dongtai_common/models/assetv2.py index e7a0e17ee..af1ed0ccb 100644 --- a/dongtai_common/models/assetv2.py +++ b/dongtai_common/models/assetv2.py @@ -99,8 +99,6 @@ class AssetV2Global(models.Model): null=False, default='') language_id = models.IntegerField(default=1, blank=True, null=False) - - safe_version_list = models.JSONField(blank=True, null=True, default=list) license_list = models.JSONField(blank=True, null=True, default=list) class Meta: @@ -109,16 +107,14 @@ class Meta: class IastAssetLicense(models.Model): + """ + only for the filter + """ license_id = models.IntegerField(blank=True, null=True) - license = models.CharField(max_length=64, - blank=True, - null=False, - default='') asset = models.ForeignKey(AssetV2Global, on_delete=models.DO_NOTHING, db_constraint=False, db_column='asset_id') - create_time = models.IntegerField(blank=True, null=True) class Meta: managed = get_managed() From 28b2b33b3f557dee2cd4392c24ec198515e4bdfd Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Tue, 30 May 2023 14:09:21 +0800 Subject: [PATCH 03/10] feat: change location. --- dongtai_common/models/asset_vul_v2.py | 13 +++---- dongtai_common/models/assetv2.py | 34 ++++++----------- dongtai_web/dongtai_sca/views/newpackage.py | 41 +++++++++++++++++---- 3 files changed, 50 insertions(+), 38 deletions(-) diff --git a/dongtai_common/models/asset_vul_v2.py b/dongtai_common/models/asset_vul_v2.py index e071e6ec8..72f7fbf11 100644 --- a/dongtai_common/models/asset_vul_v2.py +++ b/dongtai_common/models/asset_vul_v2.py @@ -13,7 +13,6 @@ class IastAssetVulV2(models.Model): - vul_id = models.CharField(max_length=255, blank=True, null=True) vul_name = models.CharField(max_length=255, blank=True, null=True) vul_detail = models.TextField(blank=True, null=True) # 漏洞类型等级 @@ -21,13 +20,12 @@ class IastAssetVulV2(models.Model): models.DO_NOTHING, blank=True, null=True) - vul_type = models.CharField(max_length=255, blank=True, null=True) - package_language = models.CharField(max_length=10, blank=True, null=True) - published_time = models.IntegerField(blank=True, null=True) - change_time = models.IntegerField(blank=True, null=True) update_time = models.IntegerField(blank=True, null=True) create_time = models.IntegerField(blank=True, null=True) references = models.JSONField(blank=True, null=True, default=list) + change_time = models.IntegerField(blank=True, null=True) + published_time = models.IntegerField(blank=True, null=True) + vul_id = models.CharField(max_length=255, blank=True, null=True) asset = models.ManyToManyField('IastVulAssetRelationV2') class Meta: @@ -40,11 +38,10 @@ class IastVulAssetRelationV2(models.Model): on_delete=models.DO_NOTHING, db_constraint=False, db_column='vul_id') - assee = models.ForeignKey(AssetV2Global, + asset = models.ForeignKey(AssetV2Global, on_delete=models.DO_NOTHING, db_constraint=False, - db_column='aql') - create_time = models.IntegerField(blank=True, null=True) + db_column='asset') class Meta: managed = get_managed() diff --git a/dongtai_common/models/assetv2.py b/dongtai_common/models/assetv2.py index af1ed0ccb..777397db8 100644 --- a/dongtai_common/models/assetv2.py +++ b/dongtai_common/models/assetv2.py @@ -30,23 +30,13 @@ class AssetV2(models.Model): id = models.BigAutoField(primary_key=True) package_name = models.CharField(max_length=255, blank=True, null=True) - aql = models.ForeignKey('AssetV2Global', - to_field='aql', - default='', - on_delete=models.DO_NOTHING) package_path = models.CharField(max_length=255, blank=True, null=True) signature_algorithm = models.CharField(max_length=255, blank=True, null=True) signature_value = models.CharField(max_length=255, blank=True, null=True) dt = models.IntegerField(blank=True, null=True) - is_reconized = models.IntegerField(blank=True, null=True) version = models.CharField(max_length=255, blank=True, null=True) - level = models.ForeignKey(IastVulLevel, - models.DO_NOTHING, - blank=True, - null=True, - default=4) project = models.ForeignKey(IastProject, on_delete=models.CASCADE, blank=True, @@ -57,17 +47,19 @@ class AssetV2(models.Model): blank=True, null=False, default=-1) - language = models.CharField(max_length=32, - blank=True, - null=False, - default='') - # 部门id department = models.ForeignKey(Department, models.DO_NOTHING, blank=True, null=True, default=-1) + language_id = models.IntegerField(default=1, blank=True, null=False) + is_reconized = models.IntegerField(blank=True, null=True) + aql = models.ForeignKey('AssetV2Global', + to_field='aql', + default='', + db_column="aql", + on_delete=models.DO_NOTHING) class Meta: managed = get_managed() @@ -76,7 +68,6 @@ class Meta: class AssetV2Global(models.Model): id = models.BigAutoField(primary_key=True) - aql = models.CharField(max_length=255, blank=True, null=True) package_name = models.CharField(max_length=255, blank=True, null=True) signature_algorithm = models.CharField(max_length=255, blank=True, @@ -94,16 +85,13 @@ class AssetV2Global(models.Model): vul_medium_count = models.IntegerField(default=0, blank=True, null=False) vul_low_count = models.IntegerField(default=0, blank=True, null=False) vul_info_count = models.IntegerField(default=0, blank=True, null=False) - language = models.CharField(max_length=32, - blank=True, - null=False, - default='') - language_id = models.IntegerField(default=1, blank=True, null=False) license_list = models.JSONField(blank=True, null=True, default=list) + language_id = models.IntegerField(default=1, blank=True, null=False) + aql = models.CharField(max_length=255, blank=True, null=True) class Meta: managed = get_managed() - db_table = 'iast_asset_v2_summary' + db_table = 'iast_asset_v2_global' class IastAssetLicense(models.Model): @@ -114,7 +102,7 @@ class IastAssetLicense(models.Model): asset = models.ForeignKey(AssetV2Global, on_delete=models.DO_NOTHING, db_constraint=False, - db_column='asset_id') + db_column='asset') class Meta: managed = get_managed() diff --git a/dongtai_web/dongtai_sca/views/newpackage.py b/dongtai_web/dongtai_sca/views/newpackage.py index 8eaf29053..d23ebb68f 100644 --- a/dongtai_web/dongtai_sca/views/newpackage.py +++ b/dongtai_web/dongtai_sca/views/newpackage.py @@ -7,6 +7,7 @@ from django.core.paginator import Paginator from django.forms.models import model_to_dict from dongtai_common.endpoint import R, UserEndPoint +from django.db.models import Q, F from django.utils.translation import gettext_lazy as _ from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer from rest_framework import serializers @@ -22,17 +23,20 @@ class PackageListArgsSerializer(serializers.Serializer): help_text=_('Number per page')) page = serializers.IntegerField(default=1, help_text=_('Page index')) language_ids = serializers.ListField( - child=serializers.IntegerField(default=1, help_text=_('language'))) + required=False, + child=serializers.IntegerField(help_text=_('language'))) license_ids = serializers.ListField( - child=serializers.IntegerField(default=1, help_text=_('license'))) + required=False, child=serializers.IntegerField(help_text=_('license'))) level_ids = serializers.ListField( - child=serializers.IntegerField(default=1, help_text=_('level'))) - project_id = serializers.IntegerField(default=1, help_text=_('Page index')) - project_version_id = serializers.IntegerField(default=1, + required=False, child=serializers.IntegerField(help_text=_('level'))) + project_id = serializers.IntegerField(required=False, + help_text=_('Page index')) + project_version_id = serializers.IntegerField(required=False, help_text=_('Page index')) - keyword = serializers.CharField(help_text=_("search_keyword")) + keyword = serializers.CharField(required=False, + help_text=_("search_keyword")) order_field = serializers.ChoiceField(['vul_count', 'level'], - default='level') + default='vul_count') order = serializers.ChoiceField(['desc', 'asc'], default='desc') @@ -52,4 +56,27 @@ class PackageList(UserEndPoint): @extend_schema_with_envcheck_v2(request=PackageListArgsSerializer, responses={200: _NewResponseSerializer}) def post(self, request): + ser = PackageListArgsSerializer(data=request.POST) + try: + if ser.is_valid(True): + pass + except ValidationError as e: + return R.failure(data=e.detail) + q = Q() + if 'level_id' in ser.validated_data: + q = q & Q(level_id__in=ser.validated_data['level_id']) + if 'language_id' in ser.validated_data: + q = q & Q(language_id__in=ser.validated_data['language_id']) + if 'license_id' in ser.validated_data: + q = q & Q(license_id__in=ser.validated_data['license_id']) + if 'project_id' in ser.validated_data: + q = q & Q(assetv2__project_id=ser.validated_data['project_id']) + if 'project_version_id' in ser.validated_data: + q = q & Q(assetv2__project_version_id=ser. + validated_data['project_version_id']) + if 'keyword' in ser.validated_data: + q = q & Q(aql__contains=ser.validated_data['keyword']) + order = '-' if ser.validated_data[ + 'order'] == 'desc' else '' + ser.validated_data['order_field'] + AssetV2Global.objects.filter() return JsonResponse({}) From a3212597dca57e89b2dc00f75600297ac57c7ea4 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Tue, 30 May 2023 19:02:55 +0800 Subject: [PATCH 04/10] feat: new sca impl. --- dongtai_common/models/asset_vul_v2.py | 29 ++++++++--------- dongtai_common/models/assetv2.py | 20 ++++++++++-- dongtai_web/dongtai_sca/urls.py | 15 +++++---- dongtai_web/dongtai_sca/views/newpackage.py | 8 +++-- .../dongtai_sca/views/newpackagedetail.py | 24 +++++++++++--- .../dongtai_sca/views/newpackageprojects.py | 21 ++++++++++--- .../views/newpackageprojectversions.py | 2 +- .../dongtai_sca/views/newpackagesummary.py | 31 ++++++++++++++----- .../dongtai_sca/views/newpackagevuldetail.py | 3 +- .../dongtai_sca/views/newpackagevuls.py | 21 ++++++++++--- 10 files changed, 128 insertions(+), 46 deletions(-) diff --git a/dongtai_common/models/asset_vul_v2.py b/dongtai_common/models/asset_vul_v2.py index 72f7fbf11..c09490afa 100644 --- a/dongtai_common/models/asset_vul_v2.py +++ b/dongtai_common/models/asset_vul_v2.py @@ -26,7 +26,6 @@ class IastAssetVulV2(models.Model): change_time = models.IntegerField(blank=True, null=True) published_time = models.IntegerField(blank=True, null=True) vul_id = models.CharField(max_length=255, blank=True, null=True) - asset = models.ManyToManyField('IastVulAssetRelationV2') class Meta: managed = True @@ -37,25 +36,27 @@ class IastVulAssetRelationV2(models.Model): asset_vul = models.ForeignKey(IastAssetVulV2, on_delete=models.DO_NOTHING, db_constraint=False, - db_column='vul_id') + db_column='vul_id', + to_field="vul_id") asset = models.ForeignKey(AssetV2Global, on_delete=models.DO_NOTHING, db_constraint=False, - db_column='asset') + db_column='asset', + to_field="aql") class Meta: managed = get_managed() db_table = 'iast_asset_vul_v2_relation' -class IastPackageGAInfo(models.Model): - package_name = models.ForeignKey(AssetV2Global, - on_delete=models.DO_NOTHING, - db_constraint=False, - db_column='package_name') - affected_versions = models.JSONField(blank=True, null=True, default=list) - unaffected_versions = models.JSONField(blank=True, null=True, default=list) - - class Meta: - managed = get_managed() - db_table = 'iast_asset_v2_ga_info' +#class IastPackageGAInfo(models.Model): +# package_name = models.ForeignKey(AssetV2Global, +# on_delete=models.DO_NOTHING, +# db_constraint=False, +# db_column='package_name') +# affected_versions = models.JSONField(blank=True, null=True, default=list) +# unaffected_versions = models.JSONField(blank=True, null=True, default=list) +# +# class Meta: +# managed = get_managed() +# db_table = 'iast_asset_v2_ga_info' diff --git a/dongtai_common/models/assetv2.py b/dongtai_common/models/assetv2.py index 777397db8..04e7936d5 100644 --- a/dongtai_common/models/assetv2.py +++ b/dongtai_common/models/assetv2.py @@ -68,7 +68,11 @@ class Meta: class AssetV2Global(models.Model): id = models.BigAutoField(primary_key=True) - package_name = models.CharField(max_length=255, blank=True, null=True) + package_name = models.ForeignKey('IastPackageGAInfo', + on_delete=models.DO_NOTHING, + db_constraint=False, + db_column='package_name', + to_field="package_name") signature_algorithm = models.CharField(max_length=255, blank=True, null=True) @@ -102,8 +106,20 @@ class IastAssetLicense(models.Model): asset = models.ForeignKey(AssetV2Global, on_delete=models.DO_NOTHING, db_constraint=False, - db_column='asset') + db_column='asset', + to_field='aql') class Meta: managed = get_managed() db_table = 'iast_asset_v2_license' + +class IastPackageGAInfo(models.Model): + package_name = models.CharField(max_length=255, + blank=True, + null=True) + affected_versions = models.JSONField(blank=True, null=True, default=list) + unaffected_versions = models.JSONField(blank=True, null=True, default=list) + + class Meta: + managed = get_managed() + db_table = 'iast_asset_v2_ga_info' diff --git a/dongtai_web/dongtai_sca/urls.py b/dongtai_web/dongtai_sca/urls.py index fc569a9ff..f49fc9721 100644 --- a/dongtai_web/dongtai_sca/urls.py +++ b/dongtai_web/dongtai_sca/urls.py @@ -25,14 +25,17 @@ v2_urlpatterns = [ path('package/', PackageList.as_view()), - path('package//', + path('package///detail', PackageDetail.as_view()), path('package_summary/', NewPackageSummary.as_view()), - path('package_vuls/', NewPackageVuls.as_view()), - path('package_vul_detail/', PackageVulDetail.as_view()), - path('package_realtion_project/', NewPackageRelationProject.as_view()), - path('package_realtion_project_version/', - NewPackageRelationProjectVersion.as_view()), + path('package_vuls//', + NewPackageVuls.as_view()), + path('package_vul/', PackageVulDetail.as_view()), + path('package///relation_projects', + NewPackageRelationProject.as_view()), + path( + 'package///relation_project/', + NewPackageRelationProjectVersion.as_view()), ] urlpatterns = [ diff --git a/dongtai_web/dongtai_sca/views/newpackage.py b/dongtai_web/dongtai_sca/views/newpackage.py index d23ebb68f..16cdc95f1 100644 --- a/dongtai_web/dongtai_sca/views/newpackage.py +++ b/dongtai_web/dongtai_sca/views/newpackage.py @@ -14,6 +14,7 @@ from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.assetv2 import AssetV2, AssetV2Global +from rest_framework.serializers import ValidationError logger = logging.getLogger(__name__) @@ -78,5 +79,8 @@ def post(self, request): q = q & Q(aql__contains=ser.validated_data['keyword']) order = '-' if ser.validated_data[ 'order'] == 'desc' else '' + ser.validated_data['order_field'] - AssetV2Global.objects.filter() - return JsonResponse({}) + page_info, data = self.get_paginator( + AssetV2Global.objects.filter(q).order_by(order).values().all(), + ser.validated_data['page'], ser.validated_data['page_size']) + return R.success(data=PackeageScaAssetSerializer(data, many=True), + page=page_info) diff --git a/dongtai_web/dongtai_sca/views/newpackagedetail.py b/dongtai_web/dongtai_sca/views/newpackagedetail.py index ff5766c0d..360b99847 100644 --- a/dongtai_web/dongtai_sca/views/newpackagedetail.py +++ b/dongtai_web/dongtai_sca/views/newpackagedetail.py @@ -12,7 +12,7 @@ from rest_framework import serializers from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id -from dongtai_common.models.assetv2 import AssetV2, AssetV2Global +from dongtai_common.models.assetv2 import (AssetV2, AssetV2Global, IastPackageGAInfo,) logger = logging.getLogger(__name__) @@ -35,18 +35,34 @@ class PackageListArgsSerializer(serializers.Serializer): order = serializers.CharField(help_text=_("order")) -class PackeageScaAssetSerializer(serializers.ModelSerializer): +class PackeageScaPlainSerializer(serializers.ModelSerializer): class Meta: model = AssetV2Global fields = '__all__' -_NewResponseSerializer = get_response_serializer(PackeageScaAssetSerializer()) +class PackeageScaAssetDetailSerializer(serializers.ModelSerializer): + affected_versions = serializers.ListField( + source='package_name.affected_versions') + unaffected_versions = serializers.ListField( + source='package_name.unaffected_versions') + + class Meta: + model = AssetV2Global + fields = '__all__' + + +_NewResponseSerializer = get_response_serializer( + PackeageScaAssetDetailSerializer()) class PackageDetail(UserEndPoint): @extend_schema_with_envcheck_v2(responses={200: _NewResponseSerializer}) def get(self, request, package_name, package_version): - return JsonResponse({}) + asset = AssetV2Global.objects.filter(package_name=package_name, + version=package_version).first() + if asset: + return R.success(data=PackeageScaAssetDetailSerializer(asset).data) + return R.failure() diff --git a/dongtai_web/dongtai_sca/views/newpackageprojects.py b/dongtai_web/dongtai_sca/views/newpackageprojects.py index bf579fbe5..d42dc49b2 100644 --- a/dongtai_web/dongtai_sca/views/newpackageprojects.py +++ b/dongtai_web/dongtai_sca/views/newpackageprojects.py @@ -39,10 +39,11 @@ class RelationProjectArgsSerializer(serializers.Serializer): help_text=_("project with be the first")) -class RelationProjectSerializer(DataclassSerializer): +class RelationProjectSerializer(serializers.ModelSerializer): class Meta: - dataclass = RelationProject + model = AssetV2 + fields = '__all__' FullRelationProjectResponseSerializer = get_response_serializer( @@ -54,5 +55,17 @@ class NewPackageRelationProject(UserEndPoint): @extend_schema_with_envcheck_v2( request=RelationProjectArgsSerializer, responses={200: FullRelationProjectResponseSerializer}) - def post(self, request): - return JsonResponse({}) + def get(self, request, package_name, package_version): + ser = RelationProjectArgsSerializer(data=request.query_params) + try: + if ser.is_valid(True): + pass + except ValidationError as e: + return R.failure(data=e.detail) + assets = AssetV2.objects.filter(package_name=package_name, + version=package_version).all() + page_info, data = self.get_paginator(assets, + ser.validated_data['page'], + ser.validated_data['page_size']) + return R.success(data=RelationProjectSerializer(data, many=True), + page=page_info) diff --git a/dongtai_web/dongtai_sca/views/newpackageprojectversions.py b/dongtai_web/dongtai_sca/views/newpackageprojectversions.py index 6ef2d8c29..7f9563aa9 100644 --- a/dongtai_web/dongtai_sca/views/newpackageprojectversions.py +++ b/dongtai_web/dongtai_sca/views/newpackageprojectversions.py @@ -52,5 +52,5 @@ class NewPackageRelationProjectVersion(UserEndPoint): @extend_schema_with_envcheck_v2( request=RelationProjectVersionArgsSerializer, responses={200: FullRelationProjectVersionResponseSerializer}) - def post(self, request): + def get(self, request, package_name, package_version, project_id): return JsonResponse({}) diff --git a/dongtai_web/dongtai_sca/views/newpackagesummary.py b/dongtai_web/dongtai_sca/views/newpackagesummary.py index 7f4b5e587..1bf3e403f 100644 --- a/dongtai_web/dongtai_sca/views/newpackagesummary.py +++ b/dongtai_web/dongtai_sca/views/newpackagesummary.py @@ -19,6 +19,7 @@ from typing import Any from dataclasses import dataclass import json +from django.db.models import Q, F, Count logger = logging.getLogger(__name__) @@ -52,12 +53,6 @@ class Data: class PackageSummaryArgsSerializer(serializers.Serializer): - language_ids = serializers.ListField( - child=serializers.IntegerField(default=1, help_text=_('language'))) - license_ids = serializers.ListField( - child=serializers.IntegerField(default=1, help_text=_('license'))) - level_ids = serializers.ListField( - child=serializers.IntegerField(default=1, help_text=_('level'))) project_id = serializers.IntegerField(default=1, help_text=_('Page index')) project_version_id = serializers.IntegerField(default=1, help_text=_('Page index')) @@ -76,7 +71,27 @@ class Meta: class NewPackageSummary(UserEndPoint): @extend_schema_with_envcheck_v2( - request=PackageSummaryArgsSerializer, + parameters=[PackageSummaryArgsSerializer], responses={200: FullSummaryResponseSerializer}) - def post(self, request): + def get(self, request): + ser = PackageSummaryArgsSerializer(data=request.query_params) + try: + if ser.is_valid(True): + pass + except ValidationError as e: + return R.failure(data=e.detail) + if 'project_id' in ser.validated_data: + q = q & Q(assetv2__project_id=ser.validated_data['project_id']) + if 'project_version_id' in ser.validated_data: + q = q & Q(assetv2__project_version_id=ser. + validated_data['project_version_id']) + queryset = AssetV2Global.objects.filter(q) + langauge_summary_list = queryset.values('language_id').annotate( + count=Count('language_id')) + level_summary_list = queryset.values('level_id').annotate( + count=Count('level_id')) + license_summary_list = queryset.annotate( + license_id=F("iastassetlicense__license_id"), + count=Count('iastassetlicense__license_id')).values( + "license_id", "count") return JsonResponse({}) diff --git a/dongtai_web/dongtai_sca/views/newpackagevuldetail.py b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py index eda51d7c8..fa035c1d8 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevuldetail.py +++ b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py @@ -49,4 +49,5 @@ class PackageVulDetail(UserEndPoint): @extend_schema_with_envcheck_v2(responses={200: _NewResponseSerializer}) def get(self, request, vul_id): - return JsonResponse({}) + asset_vul = IastAssetVulV2.objects.filter(vul_id=vul_id).first() + return R.success(data=PackeageVulsSerializer(asset_vul), ) diff --git a/dongtai_web/dongtai_sca/views/newpackagevuls.py b/dongtai_web/dongtai_sca/views/newpackagevuls.py index 60cd5ca07..389720314 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevuls.py +++ b/dongtai_web/dongtai_sca/views/newpackagevuls.py @@ -53,8 +53,6 @@ class PackageVulsListArgsSerializer(serializers.Serializer): page_size = serializers.IntegerField(default=20, help_text=_('Number per page')) page = serializers.IntegerField(default=1, help_text=_('Page index')) - package_name = serializers.CharField(help_text=_("pacakge name")) - package_version = serializers.CharField(help_text=_("package version")) class PackeageVulsSerializer(serializers.ModelSerializer): @@ -73,5 +71,20 @@ class NewPackageVuls(UserEndPoint): @extend_schema_with_envcheck_v2( request=PackageVulsListArgsSerializer, responses={200: NewPackageVulSResponseSerializer}) - def post(self, request): - return JsonResponse({}) + def get(self, request, package_name, package_version): + ser = PackageListArgsSerializer(data=request.GET) + try: + if ser.is_valid(True): + pass + except ValidationError as e: + return R.failure(data=e.detail) + asset_vuls = IastAssetVulV2.objects.filter( + iastvulassetrelationv2__asset__package_name=pacakge_name, + iastvulassetrelationv2__asset__version=package_version).order_by( + '-id').all() + page_info, data = self.get_paginator(asset_vuls, + ser.validated_data['page'], + ser.validated_data['page_size']) + + return R.success(data=PackeageVulsSerializer(data, many=True).data, + page=page_info) From f3d0057c6477a456c06f154764b3ec680502c650 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Wed, 31 May 2023 10:48:47 +0800 Subject: [PATCH 05/10] feat: new sca impl. --- dongtai_web/dongtai_sca/scan/utils.py | 22 +++++++++++++++ .../dongtai_sca/views/newpackagesummary.py | 28 ++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/dongtai_web/dongtai_sca/scan/utils.py b/dongtai_web/dongtai_sca/scan/utils.py index a26682ba2..e335ca6db 100644 --- a/dongtai_web/dongtai_sca/scan/utils.py +++ b/dongtai_web/dongtai_sca/scan/utils.py @@ -435,6 +435,28 @@ def get_ecosystem_language_dict() -> defaultdict: }) +def get_language(language_id: int) -> str: + return defaultdict(lambda: "Java", { + 1: "Java", + 2: "Python", + 3: "PHP", + 4: "Golang", + }) + + +def get_level(level_id: int) -> str: + return defaultdict(lambda: "无风险", { + 1: "高危", + 2: "中危", + 3: "低危", + 4: "无风险", + }) + + +def get_license(license_id: int) -> str: + return defaultdict(lambda: "non-standard", {}) + + def get_description(descriptions: List[Dict]) -> str: if not descriptions: return "" diff --git a/dongtai_web/dongtai_sca/views/newpackagesummary.py b/dongtai_web/dongtai_sca/views/newpackagesummary.py index 1bf3e403f..f1fca86c1 100644 --- a/dongtai_web/dongtai_sca/views/newpackagesummary.py +++ b/dongtai_web/dongtai_sca/views/newpackagesummary.py @@ -20,6 +20,7 @@ from dataclasses import dataclass import json from django.db.models import Q, F, Count +from dongtai_web.dongtai_sca.scan.utils import get_level, get_license, get_language logger = logging.getLogger(__name__) @@ -52,6 +53,22 @@ class Data: license: List[License] +def item_data_transfrom( + summary_dict, + function, + key, + new_key, +): + summary_dict[new_key] = function(summary_dict[key]) + return summary_dict + + +def data_transfrom(dict_list, function, key, new_key): + return list( + map(lambda x: item_data_transfrom(x, function, key, new_key), + dict_list)) + + class PackageSummaryArgsSerializer(serializers.Serializer): project_id = serializers.IntegerField(default=1, help_text=_('Page index')) project_version_id = serializers.IntegerField(default=1, @@ -94,4 +111,13 @@ def get(self, request): license_id=F("iastassetlicense__license_id"), count=Count('iastassetlicense__license_id')).values( "license_id", "count") - return JsonResponse({}) + return R.success(data=DataclassSerializer(data={ + "language": + data_transfrom(language_summary_list, get_language, "language_id", + "language"), + "license": + data_transfrom(license_summary_list, get_license, "license_id", + "license"), + "level": + data_transfrom(level_summary_list, get_level, "level_id", "level"), + }, ).data) From c36bb6a57a622b929c103d4647aa6ae25981a699 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Wed, 31 May 2023 14:23:35 +0800 Subject: [PATCH 06/10] feat: new sca impl. --- Pipfile | 4 + Pipfile.lock | 158 +++++++++++++++++--- dongtai_common/models/assetv2.py | 5 +- dongtai_web/dongtai_sca/common/dataclass.py | 2 +- requirements.txt | 23 ++- 5 files changed, 163 insertions(+), 29 deletions(-) diff --git a/Pipfile b/Pipfile index 207913718..60829b528 100644 --- a/Pipfile +++ b/Pipfile @@ -79,6 +79,10 @@ uwsgi = "==2.0.21" marisa-trie = "==0.8.0" gunicorn = "==20.1.0" celery-singleton = "*" +djangorestframework-dataclasses = "*" +django-seriously = "*" +dataclasses-json = "*" +django-silk = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index f22292681..dfc762049 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "03db2a64eb4ba513dc300690d6587bce6eb10dc6d5f63fdbd7fc56f510a36b81" + "sha256": "b7f57cdde9d4de50d9fb6121dd114e3c6ee8adfc6e3792b79b1e6d0a3f879746" }, "pipfile-spec": 6, "requires": { @@ -30,10 +30,10 @@ }, "aliyun-python-sdk-kms": { "hashes": [ - "sha256:a7f185772c88f3a0dda856b666dda436d82e00f9f11ea5bbf12dcab2610ee358", - "sha256:c800a1f64d5bac2b1f3589767355f4da606ddf86164f9613d0fb2a84e65b3bea" + "sha256:9bc39c693ba83944f5dfb871b118a2925eb8a5ee214dfcce61ee2ea3b6317ef1", + "sha256:a372737715682014bace68bd40fe83332f4fd925009a3eb110d41bc66f270e7a" ], - "version": "==2.16.0" + "version": "==2.16.1" }, "amqp": { "hashes": [ @@ -45,11 +45,11 @@ }, "asgiref": { "hashes": [ - "sha256:33958cb2e4b3cd8b1b06ef295bd8605cde65b11df51d3beab39e2e149a610ab3", - "sha256:8de379fcc383bcfe4507e229fc31209ea23d4831c850f74063b2c11639474dd2" + "sha256:89b2ef2247e3b562a16eef663bc0e2e703ec6468e2fa8a5cd61cd449786d4f6e", + "sha256:9e0ce3aa93a819ba5b45120216b23878cf6e8525eb3848653452b4192b92afed" ], "index": "pypi", - "version": "==3.7.1" + "version": "==3.7.2" }, "async-timeout": { "hashes": [ @@ -67,6 +67,14 @@ "markers": "python_version >= '3.7'", "version": "==23.1.0" }, + "autopep8": { + "hashes": [ + "sha256:86e9303b5e5c8160872b2f5ef611161b2893e9bfe8ccc7e2f76385947d57a2f1", + "sha256:f9849cdd62108cb739dbcdbfb7fdcc9a30d1b63c4cc3e1c1f893b5360941b61c" + ], + "markers": "python_version >= '3.6'", + "version": "==2.0.2" + }, "billiard": { "hashes": [ "sha256:0f50d6be051c6b2b75bfbc8bfd85af195c5739c281d3f5b86a5640c65563614a", @@ -85,11 +93,11 @@ }, "boto3-stubs": { "hashes": [ - "sha256:3295cc42aae7cd52f2cb39eb5c95caf13391e17b618c0c20d10aa80435991233", - "sha256:de05f2e545ff46d10c2edfbad2805871b2dd1313a9cccc64e48c5d08e977bb5a" + "sha256:71956923d1b80ba3401426900edf4ffcbeb77652d83b3c2b65b299861d2f981e", + "sha256:b7d4115689e35282b57414c07cff7a1ade0b1f8e37036fe86a04e53e0731d0d4" ], "index": "pypi", - "version": "==1.26.141" + "version": "==1.26.143" }, "botocore": { "hashes": [ @@ -101,11 +109,11 @@ }, "botocore-stubs": { "hashes": [ - "sha256:6d62f82692b4e25456962a00028402c68ad541dd2a3619b57605959550a2a81e", - "sha256:867889a4dc3eae9a479a3720accfefa03324f5415fe199e44d996b28a1bf097b" + "sha256:ba2e3d41bb55dbc298b46cabdf52175e37ce9686cb7c9ff0c24e7212452c4efb", + "sha256:bc978b87ad3ef591389e8da2eda902958c90204c1e573be36232dfa7dd120f25" ], "index": "pypi", - "version": "==1.29.141" + "version": "==1.29.143" }, "celery": { "hashes": [ @@ -358,6 +366,14 @@ "index": "pypi", "version": "==39.0.1" }, + "dataclasses-json": { + "hashes": [ + "sha256:bc285b5f892094c3a53d558858a88553dd6a61a11ab1a8128a0e554385dcc5dd", + "sha256:c2c11bc8214fbf709ffc369d11446ff6945254a7f09128154a7620613d8fda90" + ], + "index": "pypi", + "version": "==0.5.7" + }, "ddt": { "hashes": [ "sha256:e3c93b961a108b4f4d5a6c7f2263513d928baf3bb5b32af8e1c804bfb041141d", @@ -473,6 +489,22 @@ "index": "pypi", "version": "==1.6.0" }, + "django-seriously": { + "hashes": [ + "sha256:3143c6d397ac82803a8890247c52466a70863c8dbb59065888ebd2c6d10cec8c", + "sha256:e20e06b33894cfd8d58370aca9a0c8a991080d71186a841454a16dc192b66a53" + ], + "index": "pypi", + "version": "==0.4.0" + }, + "django-silk": { + "hashes": [ + "sha256:2f1fcaaf21192011147537fe1ca72dc9f552f32d7043ebd82aeeda370f194469", + "sha256:50552f06d9306d06517fbeab9a2c74856355e06304f03ed16b6dd353f7c77e7a" + ], + "index": "pypi", + "version": "==5.0.3" + }, "django-simple-captcha": { "hashes": [ "sha256:9649e66dab4e71efacbfef02f48b83b91684898352a1ab56f1686ce71033b328", @@ -539,6 +571,14 @@ "index": "pypi", "version": "==3.12.4" }, + "djangorestframework-dataclasses": { + "hashes": [ + "sha256:4fa8d93a19a3e256487674e9c654b45992bc76de155bd097eced6fe4af4100a2", + "sha256:e4f54a2a21b6dfdb07ba657f91b5a97a41d03c6760203e8a6c3749276d3f129c" + ], + "index": "pypi", + "version": "==1.2.0" + }, "djangorestframework-stubs": { "extras": [ "compatible-mypy" @@ -655,6 +695,14 @@ "index": "pypi", "version": "==22.10.2" }, + "gprof2dot": { + "hashes": [ + "sha256:45b4d298bd36608fccf9511c3fd88a773f7a1abc04d6cd39445b11ba43133ec5", + "sha256:f165b3851d3c52ee4915eb1bd6cca571e5759823c2cd0f71a79bda93c2dc85d6" + ], + "markers": "python_version >= '2.7'", + "version": "==2022.7.29" + }, "greenlet": { "hashes": [ "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a", @@ -847,11 +895,11 @@ }, "kombu": { "hashes": [ - "sha256:1d565fd1aad51f7bca86ba35dee7857303007de64a4500b8a86d83634380c487", - "sha256:fd0f2665552109ed980249e97ab91b730690b073bc417481af0a8228d81620c7" + "sha256:10f17896f9d5dc28b80a882badbd333931c77c82aaa45cd4b84fcdc74398ff2b", + "sha256:8e10da0d5c5823dbfc0c30b5b5edb2f3b0e64c2a8378eb19d0bc0425b9c88f6f" ], "markers": "python_version >= '3.8'", - "version": "==5.3.0rc1" + "version": "==5.3.0rc2" }, "lxml": { "hashes": [ @@ -1060,6 +1108,21 @@ "markers": "python_version >= '3.7'", "version": "==2.1.2" }, + "marshmallow": { + "hashes": [ + "sha256:90032c0fd650ce94b6ec6dc8dfeb0e3ff50c144586462c389b81a07205bedb78", + "sha256:93f0958568da045b0021ec6aeb7ac37c81bfcccbb9a0e7ed8559885070b3a19b" + ], + "markers": "python_version >= '3.7'", + "version": "==3.19.0" + }, + "marshmallow-enum": { + "hashes": [ + "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58", + "sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072" + ], + "version": "==1.5.1" + }, "mock": { "hashes": [ "sha256:06f18d7d65b44428202b145a9a36e99c2ee00d1eb992df0caf881d4664377891", @@ -1276,6 +1339,14 @@ "markers": "python_full_version >= '3.7.0'", "version": "==3.0.38" }, + "pycodestyle": { + "hashes": [ + "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", + "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" + ], + "markers": "python_version >= '3.6'", + "version": "==2.10.0" + }, "pycparser": { "hashes": [ "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", @@ -1354,6 +1425,48 @@ "index": "pypi", "version": "==3.14.1" }, + "pydantic": { + "hashes": [ + "sha256:052d8654cb65174d6f9490cc9b9a200083a82cf5c3c5d3985db765757eb3b375", + "sha256:0c6fafa0965b539d7aab0a673a046466d23b86e4b0e8019d25fd53f4df62c277", + "sha256:1243d28e9b05003a89d72e7915fdb26ffd1d39bdd39b00b7dbe4afae4b557f9d", + "sha256:12f7b0bf8553e310e530e9f3a2f5734c68699f42218bf3568ef49cd9b0e44df4", + "sha256:1410275520dfa70effadf4c21811d755e7ef9bb1f1d077a21958153a92c8d9ca", + "sha256:16f8c3e33af1e9bb16c7a91fc7d5fa9fe27298e9f299cff6cb744d89d573d62c", + "sha256:17aef11cc1b997f9d574b91909fed40761e13fac438d72b81f902226a69dac01", + "sha256:191ba419b605f897ede9892f6c56fb182f40a15d309ef0142212200a10af4c18", + "sha256:1952526ba40b220b912cdc43c1c32bcf4a58e3f192fa313ee665916b26befb68", + "sha256:1ced8375969673929809d7f36ad322934c35de4af3b5e5b09ec967c21f9f7887", + "sha256:2e4148e635994d57d834be1182a44bdb07dd867fa3c2d1b37002000646cc5459", + "sha256:34d327c81e68a1ecb52fe9c8d50c8a9b3e90d3c8ad991bfc8f953fb477d42fb4", + "sha256:35db5301b82e8661fa9c505c800d0990bc14e9f36f98932bb1d248c0ac5cada5", + "sha256:3e59417ba8a17265e632af99cc5f35ec309de5980c440c255ab1ca3ae96a3e0e", + "sha256:42aa0c4b5c3025483240a25b09f3c09a189481ddda2ea3a831a9d25f444e03c1", + "sha256:666bdf6066bf6dbc107b30d034615d2627e2121506c555f73f90b54a463d1f33", + "sha256:66a703d1983c675a6e0fed8953b0971c44dba48a929a2000a493c3772eb61a5a", + "sha256:6a82d6cda82258efca32b40040228ecf43a548671cb174a1e81477195ed3ed56", + "sha256:6f2e754d5566f050954727c77f094e01793bcb5725b663bf628fa6743a5a9108", + "sha256:7456eb22ed9aaa24ff3e7b4757da20d9e5ce2a81018c1b3ebd81a0b88a18f3b2", + "sha256:7b1f6cb446470b7ddf86c2e57cd119a24959af2b01e552f60705910663af09a4", + "sha256:7d5b8641c24886d764a74ec541d2fc2c7fb19f6da2a4001e6d580ba4a38f7878", + "sha256:84d80219c3f8d4cad44575e18404099c76851bc924ce5ab1c4c8bb5e2a2227d0", + "sha256:88f195f582851e8db960b4a94c3e3ad25692c1c1539e2552f3df7a9e972ef60e", + "sha256:93e6bcfccbd831894a6a434b0aeb1947f9e70b7468f274154d03d71fabb1d7c6", + "sha256:93e766b4a8226e0708ef243e843105bf124e21331694367f95f4e3b4a92bbb3f", + "sha256:ab523c31e22943713d80d8d342d23b6f6ac4b792a1e54064a8d0cf78fd64e800", + "sha256:bb14388ec45a7a0dc429e87def6396f9e73c8c77818c927b6a60706603d5f2ea", + "sha256:c0ab53b609c11dfc0c060d94335993cc2b95b2150e25583bec37a49b2d6c6c3f", + "sha256:c33b60054b2136aef8cf190cd4c52a3daa20b2263917c49adad20eaf381e823b", + "sha256:ceb6a23bf1ba4b837d0cfe378329ad3f351b5897c8d4914ce95b85fba96da5a1", + "sha256:d532bf00f381bd6bc62cabc7d1372096b75a33bc197a312b03f5838b4fb84edd", + "sha256:df7800cb1984d8f6e249351139667a8c50a379009271ee6236138a22a0c0f319", + "sha256:e82d4566fcd527eae8b244fa952d99f2ca3172b7e97add0b43e2d97ee77f81ab", + "sha256:f90c1e29f447557e9e26afb1c4dbf8768a10cc676e3781b6a577841ade126b85", + "sha256:f9613fadad06b4f3bc5db2653ce2f22e0de84a7c6c293909b48f6ed37b83c61f" + ], + "markers": "python_version >= '3.7'", + "version": "==1.10.8" + }, "pymysql": { "hashes": [ "sha256:41fc3a0c5013d5f039639442321185532e3e2c8924687abe6537de157d403641", @@ -1643,11 +1756,11 @@ }, "types-requests": { "hashes": [ - "sha256:7c5cea7940f8e92ec560bbc468f65bf684aa3dcf0554a6f8c4710f5f708dc598", - "sha256:c1c29d20ab8d84dff468d7febfe8e0cb0b4664543221b386605e14672b44ea25" + "sha256:3de667cffa123ce698591de0ad7db034a5317457a596eb0b4944e5a9d9e8d1ac", + "sha256:afb06ef8f25ba83d59a1d424bd7a5a939082f94b94e90ab5e6116bd2559deaa3" ], "index": "pypi", - "version": "==2.31.0.0" + "version": "==2.31.0.1" }, "types-s3transfer": { "hashes": [ @@ -1673,6 +1786,13 @@ "index": "pypi", "version": "==4.6.2" }, + "typing-inspect": { + "hashes": [ + "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", + "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78" + ], + "version": "==0.9.0" + }, "tzdata": { "hashes": [ "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a", diff --git a/dongtai_common/models/assetv2.py b/dongtai_common/models/assetv2.py index 04e7936d5..a7c011a4a 100644 --- a/dongtai_common/models/assetv2.py +++ b/dongtai_common/models/assetv2.py @@ -113,10 +113,9 @@ class Meta: managed = get_managed() db_table = 'iast_asset_v2_license' + class IastPackageGAInfo(models.Model): - package_name = models.CharField(max_length=255, - blank=True, - null=True) + package_name = models.CharField(max_length=255, blank=True, null=True) affected_versions = models.JSONField(blank=True, null=True, default=list) unaffected_versions = models.JSONField(blank=True, null=True, default=list) diff --git a/dongtai_web/dongtai_sca/common/dataclass.py b/dongtai_web/dongtai_sca/common/dataclass.py index d5efc101f..0313a8c85 100644 --- a/dongtai_web/dongtai_sca/common/dataclass.py +++ b/dongtai_web/dongtai_sca/common/dataclass.py @@ -72,6 +72,7 @@ class Data: affected_versions: List[str] = [] unaffected_versions: List[str] = [] + @dataclass_json @dataclass class PackageInfo: @@ -82,4 +83,3 @@ class PackageInfo: hash: str license: List[str] version_publish_time: str - diff --git a/requirements.txt b/requirements.txt index 92432d647..60c2a1280 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,17 @@ -i https://pypi.org/simple aliyun-python-sdk-core==2.13.36 aliyun-python-sdk-core-v3==2.13.33 -aliyun-python-sdk-kms==2.16.0 +aliyun-python-sdk-kms==2.16.1 amqp==5.1.1 ; python_version >= '3.6' -asgiref==3.7.1 +asgiref==3.7.2 async-timeout==4.0.2 ; python_version >= '3.6' attrs==23.1.0 ; python_version >= '3.7' +autopep8==2.0.2 ; python_version >= '3.6' billiard==4.1.0 ; python_version >= '3.7' boto3==1.24.59 -boto3-stubs==1.26.141 +boto3-stubs==1.26.143 botocore==1.27.91 -botocore-stubs==1.29.141 +botocore-stubs==1.29.143 celery==5.3.0rc1 celery-singleton==0.3.1 certifi==2023.5.7 @@ -23,6 +24,7 @@ click-plugins==1.1.1 click-repl==0.2.0 crcmod==1.7 cryptography==39.0.1 +dataclasses-json==0.5.7 ddt==1.6.0 defusedxml==0.7.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' diff-match-patch==20230430 ; python_version >= '3.7' @@ -38,6 +40,8 @@ django-modeltranslation==0.17.7 django-ranged-response==0.2.0 django-redis==5.2.0 django-rest-framework-proxy==1.6.0 +django-seriously==0.4.0 +django-silk==5.0.3 django-simple-captcha==0.5.17 django-stubs[compatible-mypy]==1.15.0 django-stubs-ext==4.2.0 ; python_version >= '3.7' @@ -46,6 +50,7 @@ django-utils==0.0.2 django-utils-six==2.0 django-xff==1.3.0 djangorestframework==3.12.4 +djangorestframework-dataclasses==1.2.0 djangorestframework-stubs[compatible-mypy]==1.9.1 docxcompose==1.3.4 docxtpl==0.16.0 @@ -54,6 +59,7 @@ elasticsearch==7.17.7 elasticsearch-dsl==7.4.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' et-xmlfile==1.1.0 ; python_version >= '3.6' gevent==22.10.2 +gprof2dot==2022.7.29 ; python_version >= '2.7' greenlet==2.0.2 ; platform_python_implementation == 'CPython' gunicorn==20.1.0 id-validator==1.0.20 @@ -64,11 +70,13 @@ jmespath==0.10.0 ; python_version >= '2.6' and python_version not in '3.0, 3.1, jq==1.3.0 jsonlog==4.0.0 jsonschema==4.17.0 -kombu==5.3.0rc1 ; python_version >= '3.8' +kombu==5.3.0rc2 ; python_version >= '3.8' lxml==4.9.1 marisa-trie==0.8.0 markuppy==1.14 markupsafe==2.1.2 ; python_version >= '3.7' +marshmallow==3.19.0 ; python_version >= '3.7' +marshmallow-enum==1.5.1 mock==5.0.2 ; python_version >= '3.6' model-bakery==1.11.0 ; python_version >= '3' mypy==1.0.1 @@ -82,9 +90,11 @@ packaging==21.3 pep8==1.7.1 pillow==9.3.0 prompt-toolkit==3.0.38 ; python_full_version >= '3.7.0' +pycodestyle==2.10.0 ; python_version >= '3.6' pycparser==2.21 pycryptodome==3.18.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pycryptodomex==3.14.1 +pydantic==1.10.8 ; python_version >= '3.7' pymysql==1.0.2 pyparsing==3.0.9 ; python_full_version >= '3.6.8' pyre2==0.3.6 @@ -111,10 +121,11 @@ types-pyopenssl==23.1.0.3 types-pytz==2023.3.0.0 types-pyyaml==6.0.12.10 types-redis==4.5.5.2 -types-requests==2.31.0.0 +types-requests==2.31.0.1 types-s3transfer==0.6.1 ; python_version >= '3.7' and python_version < '4.0' types-urllib3==1.26.25.13 typing-extensions==4.6.2 +typing-inspect==0.9.0 tzdata==2023.3 ; python_version >= '2' uritemplate==4.1.1 ; python_version >= '3.6' urllib3==1.26.5 From 7b6f18266783ed4b66477a78148098579a2e2992 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Wed, 31 May 2023 14:32:03 +0800 Subject: [PATCH 07/10] feat: new sca impl. --- dongtai_web/dongtai_sca/common/dataclass.py | 13 +++++++------ dongtai_web/dongtai_sca/views/newpackageprojects.py | 1 + dongtai_web/dongtai_sca/views/newpackagesummary.py | 4 +++- .../dongtai_sca/views/newpackagevuldetail.py | 2 +- dongtai_web/dongtai_sca/views/newpackagevuls.py | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/dongtai_web/dongtai_sca/common/dataclass.py b/dongtai_web/dongtai_sca/common/dataclass.py index 0313a8c85..60bcc8898 100644 --- a/dongtai_web/dongtai_sca/common/dataclass.py +++ b/dongtai_web/dongtai_sca/common/dataclass.py @@ -12,12 +12,6 @@ class Reference: url: str = "" -@dataclass_json -@dataclass -class Root: - status: int - msg: str - data: Data @dataclass_json @@ -83,3 +77,10 @@ class PackageInfo: hash: str license: List[str] version_publish_time: str + +@dataclass_json +@dataclass +class Root: + status: int + msg: str + data: Data diff --git a/dongtai_web/dongtai_sca/views/newpackageprojects.py b/dongtai_web/dongtai_sca/views/newpackageprojects.py index d42dc49b2..a4984b84b 100644 --- a/dongtai_web/dongtai_sca/views/newpackageprojects.py +++ b/dongtai_web/dongtai_sca/views/newpackageprojects.py @@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _ from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer from rest_framework import serializers +from rest_framework.serializers import ValidationError from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.assetv2 import AssetV2, AssetV2Global diff --git a/dongtai_web/dongtai_sca/views/newpackagesummary.py b/dongtai_web/dongtai_sca/views/newpackagesummary.py index f1fca86c1..6f9c898b4 100644 --- a/dongtai_web/dongtai_sca/views/newpackagesummary.py +++ b/dongtai_web/dongtai_sca/views/newpackagesummary.py @@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _ from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer from rest_framework import serializers +from rest_framework.serializers import ValidationError from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.assetv2 import AssetV2, AssetV2Global @@ -97,13 +98,14 @@ def get(self, request): pass except ValidationError as e: return R.failure(data=e.detail) + q = Q() if 'project_id' in ser.validated_data: q = q & Q(assetv2__project_id=ser.validated_data['project_id']) if 'project_version_id' in ser.validated_data: q = q & Q(assetv2__project_version_id=ser. validated_data['project_version_id']) queryset = AssetV2Global.objects.filter(q) - langauge_summary_list = queryset.values('language_id').annotate( + language_summary_list = queryset.values('language_id').annotate( count=Count('language_id')) level_summary_list = queryset.values('level_id').annotate( count=Count('level_id')) diff --git a/dongtai_web/dongtai_sca/views/newpackagevuldetail.py b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py index fa035c1d8..d7d414cb7 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevuldetail.py +++ b/dongtai_web/dongtai_sca/views/newpackagevuldetail.py @@ -50,4 +50,4 @@ class PackageVulDetail(UserEndPoint): @extend_schema_with_envcheck_v2(responses={200: _NewResponseSerializer}) def get(self, request, vul_id): asset_vul = IastAssetVulV2.objects.filter(vul_id=vul_id).first() - return R.success(data=PackeageVulsSerializer(asset_vul), ) + return R.success(data=PackeageVulSerializer(asset_vul), ) diff --git a/dongtai_web/dongtai_sca/views/newpackagevuls.py b/dongtai_web/dongtai_sca/views/newpackagevuls.py index 389720314..ee48e85f9 100644 --- a/dongtai_web/dongtai_sca/views/newpackagevuls.py +++ b/dongtai_web/dongtai_sca/views/newpackagevuls.py @@ -10,6 +10,7 @@ from django.utils.translation import gettext_lazy as _ from dongtai_web.utils import extend_schema_with_envcheck_v2, get_response_serializer from rest_framework import serializers +from rest_framework.serializers import ValidationError from dongtai_web.dongtai_sca.utils import get_asset_id_by_aggr_id from dongtai_common.models.asset_vul_v2 import IastAssetVulV2 @@ -72,14 +73,14 @@ class NewPackageVuls(UserEndPoint): request=PackageVulsListArgsSerializer, responses={200: NewPackageVulSResponseSerializer}) def get(self, request, package_name, package_version): - ser = PackageListArgsSerializer(data=request.GET) + ser = PackageVulsListArgsSerializer(data=request.GET) try: if ser.is_valid(True): pass except ValidationError as e: return R.failure(data=e.detail) asset_vuls = IastAssetVulV2.objects.filter( - iastvulassetrelationv2__asset__package_name=pacakge_name, + iastvulassetrelationv2__asset__package_name=package_name, iastvulassetrelationv2__asset__version=package_version).order_by( '-id').all() page_info, data = self.get_paginator(asset_vuls, From 313831c0a6f9251931c43fcec55beb17098f4744 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Wed, 31 May 2023 14:33:19 +0800 Subject: [PATCH 08/10] feat: new sca impl. --- dongtai_web/dongtai_sca/scan/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dongtai_web/dongtai_sca/scan/utils.py b/dongtai_web/dongtai_sca/scan/utils.py index e335ca6db..43fb981dd 100644 --- a/dongtai_web/dongtai_sca/scan/utils.py +++ b/dongtai_web/dongtai_sca/scan/utils.py @@ -441,7 +441,7 @@ def get_language(language_id: int) -> str: 2: "Python", 3: "PHP", 4: "Golang", - }) + })['language_id'] def get_level(level_id: int) -> str: @@ -450,11 +450,11 @@ def get_level(level_id: int) -> str: 2: "中危", 3: "低危", 4: "无风险", - }) + })[level_id] def get_license(license_id: int) -> str: - return defaultdict(lambda: "non-standard", {}) + return defaultdict(lambda: "non-standard", {})[license_id] def get_description(descriptions: List[Dict]) -> str: From 666822c0235b2c2adb29fe270e40da410784a905 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Wed, 31 May 2023 14:37:51 +0800 Subject: [PATCH 09/10] feat: new sca impl. --- dongtai_web/dongtai_sca/common/dataclass.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dongtai_web/dongtai_sca/common/dataclass.py b/dongtai_web/dongtai_sca/common/dataclass.py index 60bcc8898..ff43443dd 100644 --- a/dongtai_web/dongtai_sca/common/dataclass.py +++ b/dongtai_web/dongtai_sca/common/dataclass.py @@ -5,6 +5,7 @@ from dataclasses_json import dataclass_json, config from datetime import datetime + @dataclass_json @dataclass class Reference: @@ -12,8 +13,6 @@ class Reference: url: str = "" - - @dataclass_json @dataclass class VulCodes: From e18e6771c8035447d309dd24fb7d6446170ce265 Mon Sep 17 00:00:00 2001 From: bidaya0 Date: Wed, 31 May 2023 14:51:08 +0800 Subject: [PATCH 10/10] feat: new sca impl. --- dongtai_common/models/asset_vul_v2.py | 5 ++++- dongtai_common/models/assetv2.py | 24 ++++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/dongtai_common/models/asset_vul_v2.py b/dongtai_common/models/asset_vul_v2.py index c09490afa..2ab3d3d31 100644 --- a/dongtai_common/models/asset_vul_v2.py +++ b/dongtai_common/models/asset_vul_v2.py @@ -25,7 +25,10 @@ class IastAssetVulV2(models.Model): references = models.JSONField(blank=True, null=True, default=list) change_time = models.IntegerField(blank=True, null=True) published_time = models.IntegerField(blank=True, null=True) - vul_id = models.CharField(max_length=255, blank=True, null=True) + vul_id = models.CharField(max_length=255, + blank=True, + null=True, + unique=True) class Meta: managed = True diff --git a/dongtai_common/models/assetv2.py b/dongtai_common/models/assetv2.py index a7c011a4a..fd06d2d39 100644 --- a/dongtai_common/models/assetv2.py +++ b/dongtai_common/models/assetv2.py @@ -29,7 +29,10 @@ class AssetV2(models.Model): id = models.BigAutoField(primary_key=True) - package_name = models.CharField(max_length=255, blank=True, null=True) + package_name = models.CharField(max_length=255, + blank=True, + null=True, + unique=True) package_path = models.CharField(max_length=255, blank=True, null=True) signature_algorithm = models.CharField(max_length=255, blank=True, @@ -68,11 +71,13 @@ class Meta: class AssetV2Global(models.Model): id = models.BigAutoField(primary_key=True) - package_name = models.ForeignKey('IastPackageGAInfo', - on_delete=models.DO_NOTHING, - db_constraint=False, - db_column='package_name', - to_field="package_name") + package_name = models.ForeignKey( + 'IastPackageGAInfo', + on_delete=models.DO_NOTHING, + db_constraint=False, + db_column='package_name', + to_field="package_name", + ) signature_algorithm = models.CharField(max_length=255, blank=True, null=True) @@ -91,7 +96,7 @@ class AssetV2Global(models.Model): vul_info_count = models.IntegerField(default=0, blank=True, null=False) license_list = models.JSONField(blank=True, null=True, default=list) language_id = models.IntegerField(default=1, blank=True, null=False) - aql = models.CharField(max_length=255, blank=True, null=True) + aql = models.CharField(max_length=255, blank=True, null=True, unique=True) class Meta: managed = get_managed() @@ -115,7 +120,10 @@ class Meta: class IastPackageGAInfo(models.Model): - package_name = models.CharField(max_length=255, blank=True, null=True) + package_name = models.CharField(max_length=255, + blank=True, + null=True, + unique=True) affected_versions = models.JSONField(blank=True, null=True, default=list) unaffected_versions = models.JSONField(blank=True, null=True, default=list)