Merge pull request #1566 from st1020/feat/merge-changes

feat: merge changes

dongtai_common/engine/compatibility.py

@@ -62,7 +62,10 @@ def parse_target_value_length(target_value: str) -> int:
     if not target_value:
         return 0
     position = target_value.rfind('*')
-    len_of_origin = int(target_value[position + 1::])
+    try:
+        len_of_origin = int(target_value[position + 1::])
+    except ValueError as e:
+        return len(target_value)
     return len_of_origin
 
 

dongtai_common/models/url_blacklist.py

@@ -4,7 +4,11 @@
 from django.db.models import IntegerChoices
 from django.utils.translation import gettext_lazy as _
 from typing import List, Dict
-
+from dongtai_common.models.project import (
+    IastProject,
+    IastProjectTemplate,
+    VulValidation,
+)
 
 class TargetOperator(IntegerChoices):
     EQUAL = 1, _("等于")
@@ -28,11 +32,11 @@ class TargetScope(IntegerChoices):
 
 class State(IntegerChoices):
     ENABLE = 1, _("ENABLE")
-    DISABLE = 2, _("DISABLE")
+    DISABLE = 0, _("DISABLE")
 
 
 class IastAgentBlackRule(models.Model):
-    user = models.ForeignKey(User, models.DO_NOTHING)
+    user = models.ForeignKey(User, models.DO_NOTHING, default=-1)
     scope = models.IntegerField(
         choices=TargetScope.choices,
         blank=True,
@@ -44,6 +48,8 @@ class IastAgentBlackRule(models.Model):
         blank=True,
         null=True,
     )
+    project = models.ForeignKey(IastProject, models.CASCADE, default=-1)
+    project_template = models.ForeignKey(IastProjectTemplate, models.CASCADE, default=-1)
 
     class Meta:
         managed = get_managed()
@@ -76,9 +82,21 @@ def to_agent_rule(self) -> Dict:
 
 
 def create_blacklist_rule(target_type: TargetType, operator: TargetOperator,
-                          value: str, user_id: int, state: State):
+                          value: str, state: State, **kwargs):
+    ruledetail = IastAgentBlackRuleDetail.objects.create(
+        target_type=target_type, operator=operator, value=value)
+    rule = IastAgentBlackRule.objects.create(state=state, **kwargs)
+    ruledetail.rule = rule
+    ruledetail.save()
+
+def update_blacklist_rule(target_type: TargetType, operator: TargetOperator,
+                          value: str, user_id: int, state: State,
+                          rule_id: int):
     ruledetail = IastAgentBlackRuleDetail.objects.create(
         target_type=target_type, operator=operator, value=value)
-    rule = IastAgentBlackRule.objects.create(user_id=user_id, state=state)
+    rule = IastAgentBlackRule.objects.filter(user_id=user_id, pk=rule_id).first()
+    rule.state = state
+    rule.save()
+    rule.iastagentblackruledetail_set.all().delete()
     ruledetail.rule = rule
     ruledetail.save()

dongtai_common/translation.py

@@ -31,7 +31,7 @@ class IastVulLevelTranslationOptions(TranslationOptions):
 
 @register(IastDeployDesc)
 class IastDeployDescTranslationOptions(TranslationOptions):
-    fields = ('desc',)
+    fields = ('desc', )
 
 
 @register(IastDocument)

dongtai_conf/celery.py

@@ -92,6 +92,18 @@
 configs["DJANGO_CELERY_BEAT_TZ_AWARE"] = False
 configs["CELERY_BEAT_SCHEDULER"] = 'django_celery_beat.schedulers:DatabaseScheduler'
 
+try:
+    from dongtai_conf.celery_extend import configs as extend_config
+
+    for k, v in extend_config.items():
+        config = configs.get(k, None)
+        if isinstance(v, dict) and isinstance(config, dict):
+            config.update(v)
+        elif isinstance(v, list) and isinstance(config, list):
+            config.extend(v)
+except ImportError:
+    pass
+
 app.namespace = 'CELERY'
 app.conf.update(configs)
 

dongtai_conf/conf/config.ini.test

@@ -48,7 +48,7 @@ async_send_delay = 2
 
 [log_service]
 host = localhost
-port = 8082
+port = 8083
 
 [other]
 domain = http://localhost.domain/

dongtai_conf/settings.py

@@ -156,6 +156,13 @@ def get_installed_apps():
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
 ]
 
+try:
+    from dongtai_conf.settings_extend import MIDDLEWARE as MIDDLEWARE_EXTEND
+
+    MIDDLEWARE.extend(MIDDLEWARE_EXTEND)
+except ImportError:
+    pass
+
 XFF_TRUSTED_PROXY_DEPTH = 20
 
 CSRF_COOKIE_NAME = "DTCsrfToken"
@@ -509,11 +516,12 @@ def safe_execute(default, exception, function, *args):
 There are two authentication methods. You can obtain csrf_token and sessionid through the login process, or access the corresponding API through the user's corresponding Token.
 
 The Token method is recommended here, and users can find it in the Agent installation interface such as -H
-'Authorization: Token {token}', here is the token corresponding to the user, the token method also requires a token like this on the request header."""
+  'Authorization: Token {token}', here is the token corresponding to the user, the token method also requires a token like this on the request header."""
       ),
+    'COMPONENT_SPLIT_REQUEST':
+    True,
 }
-REST_FRAMEWORK[
-    'DEFAULT_SCHEMA_CLASS'] = 'drf_spectacular.openapi.AutoSchema'
+REST_FRAMEWORK['DEFAULT_SCHEMA_CLASS'] = 'drf_spectacular.openapi.AutoSchema'
 
 if os.getenv('environment', None) == 'TEST' or os.getenv('CPROFILE',
                                                          None) == 'TRUE':
@@ -532,11 +540,11 @@ def safe_execute(default, exception, function, *args):
     SCA_TIMEOUT = 0
     SCA_TOKEN = ""
     SCA_SETUP = False
+DOMAIN = config.get('other', 'domain', fallback="")
 
 if os.getenv('environment', None) in ('TEST', 'PROD'):
     SESSION_COOKIE_DOMAIN = config.get('other', 'demo_session_cookie_domain')
     CSRF_COOKIE_DOMAIN = SESSION_COOKIE_DOMAIN
-    DOMAIN = config.get('other', 'domain')
 
 try:
     DOMAIN_VUL = config.get('other', 'domain_vul')

dongtai_engine/plugins/strategy_headers.py

@@ -20,6 +20,7 @@
 from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability, IastHeaderVulnerabilityDetail
 from django.db import IntegrityError
 from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.signals import send_notify
 
 
 class FakeSocket():
@@ -198,6 +199,11 @@ def save_vul(vul_type, method_pool, position=None, data=None):
         )
         log_vul_found(vul.agent.user_id, vul.agent.bind_project.name,
                       vul.agent.bind_project_id, vul.id, vul.strategy.vul_name)  # type: ignore
+        send_notify.send_robust(
+            sender=save_vul,
+            vul_id=vul.id,
+            department_id=method_pool.agent.department_id,
+        )
     cache.delete(cache_key)
     header_vul = None
     if not IastHeaderVulnerability.objects.filter(

dongtai_engine/signals/__init__.py

@@ -5,4 +5,4 @@
 # project: dongtai-engine
 
 # inappropriate implementation to trigger task of load handlers
-from .signals import vul_found
+from .signals import vul_found, send_notify

dongtai_engine/signals/handlers/vul_handler.py

@@ -22,6 +22,7 @@
 from collections import defaultdict
 from dongtai_common.models.profile import IastProfile
 from dongtai_engine.plugins.project_time_update import project_time_stamp_update
+from dongtai_engine.signals import send_notify
 
 
 def equals(source, target):
@@ -356,6 +357,12 @@ def save_vul(vul_meta, vul_level, strategy_id, vul_stack, top_stack,
         )
         log_vul_found(vul.agent.user_id, vul.agent.bind_project.name,  # type: ignore
                       vul.agent.bind_project_id, vul.id, vul.strategy.vul_name)  # type: ignore
+        send_notify.send_robust(
+            sender=save_vul,
+            vul_id=vul.id,
+            department_id=vul_meta.agent.department_id,
+        )
+
     cache.delete(cache_key)
     #delete if exists more than one   departured use redis lock
     #IastVulnerabilityModel.objects.filter(

dongtai_engine/signals/signals.py

@@ -4,3 +4,4 @@
 from django.dispatch import Signal
 
 vul_found = Signal()
+send_notify = Signal()

dongtai_engine/tasks.py

@@ -34,7 +34,6 @@
 from dongtai_engine.plugins.strategy_sensitive import check_response_content
 from dongtai_engine.replay import Replay
 from dongtai_conf import settings
-from dongtai_web.dongtai_sca.utils import sca_scan_asset
 import requests
 from dongtai_engine.task_base import replay_payload_data
 from dongtai_engine.common.queryset import get_scan_id, load_sink_strategy, get_agent
@@ -297,63 +296,6 @@ def get_project_agents(agent):
     return agents
 
 
-@shared_task(queue='dongtai-sca-task')
-def update_one_sca(agent_id, package_path, package_signature, package_name, package_algorithm, package_version=''):
-    """
-    根据SCA数据库，更新SCA记录信息
-    :return:
-    """
-    logger.info(
-        f'SCA检测开始 [{agent_id} {package_path} {package_signature} {package_name} {package_algorithm} {package_version}]')
-    agent = IastAgent.objects.filter(id=agent_id).first()
-    version = package_version
-    if not version:
-        if agent.language == "JAVA":
-            version = package_name.split('/')[-1].replace('.jar', '').split('-')[-1]
-
-    if version:
-        current_version_agents = get_project_agents(agent)
-        if package_signature:
-            asset_count = Asset.objects.values("id").filter(signature_value=package_signature,
-                                                            agent__in=current_version_agents).count()
-        else:
-            package_signature = sha_1(package_name)
-            asset_count = Asset.objects.values("id").filter(package_name=package_name,
-                                                            version=version,
-                                                            agent__in=current_version_agents).count()
-
-        if asset_count == 0:
-            new_level = IastVulLevel.objects.get(name="info")
-            asset = Asset()
-            asset.package_name = package_name
-            asset.package_path = package_path
-            asset.signature_value = package_signature
-            asset.signature_algorithm = package_algorithm
-            asset.version = version
-            asset.level_id = new_level.id
-            asset.vul_count = 0
-            asset.language = asset.language
-            if agent:
-                asset.agent = agent
-                asset.project_version_id = agent.project_version_id if agent.project_version_id else 0
-                asset.project_name = agent.project_name
-                asset.language = agent.language
-                asset.project_id = -1
-                if agent.bind_project_id:
-                    asset.project_id = agent.bind_project_id
-                asset.user_id = -1
-                if agent.user_id:
-                    asset.user_id = agent.user_id
-
-            asset.license = ''
-            asset.dt = int(time.time())
-            asset.save()
-            sca_scan_asset(asset)
-        else:
-            logger.info(
-                f'SCA检测开始 [{agent_id} {package_path} {package_signature} {package_name} {package_algorithm} {version}] 组件已存在')
-
-
 def sha_1(raw):
     sha1_str = hashlib.sha1(raw.encode("utf-8"), usedforsecurity=False).hexdigest()
     return sha1_str

dongtai_protocol/report/handler/hardencode_vul_handler.py

@@ -22,6 +22,7 @@
 from rest_framework.serializers import ValidationError
 from dongtai_web.vul_log.vul_log import log_vul_found
 from dongtai_common.models.agent import IastAgent
+from dongtai_engine.signals import send_notify
 
 logger = logging.getLogger('dongtai.openapi')
 
@@ -130,3 +131,8 @@ def save(self):
         log_vul_found(iast_vul.agent.user_id, iast_vul.agent.bind_project.name,
                       iast_vul.agent.bind_project_id, iast_vul.id,  # type: ignore
                       iast_vul.strategy.vul_name)
+        send_notify.send_robust(
+            sender=self.__class__,
+            vul_id=iast_vul.id,
+            department_id=self.agent.department_id,
+        )

dongtai_protocol/report/handler/narmal_vul_handler.py

@@ -22,6 +22,7 @@
 from dongtai_common.models.header_vulnerablity import IastHeaderVulnerability
 from django.db import IntegrityError
 from dongtai_protocol import utils
+from dongtai_engine.signals import send_notify
 
 logger = logging.getLogger('dongtai.openapi')
 
@@ -277,6 +278,12 @@ def save(self):
                           iast_vul.agent.bind_project.name,
                           iast_vul.agent.bind_project_id, iast_vul.id,  # type: ignore
                           iast_vul.strategy.vul_name)
+            send_notify.send_robust(
+                sender=self.__class__,
+                vul_id=iast_vul.id,
+                department_id=self.agent.department_id,
+            )
+
         IastVulnerabilityModel.objects.filter(
             strategy_id=iast_vul.strategy_id,
             uri=iast_vul.uri,

dongtai_web/aggr_vul/aggr_vul_list.py

@@ -1,4 +1,5 @@
 # 按类型获取 组件漏洞 应用漏洞列表
+from typing import Any
 from elasticsearch_dsl import Q, Search
 from dongtai_common.models.asset_vul import IastAssetVulnerabilityDocument
 from dongtai_common.common.utils import make_hash
@@ -266,9 +267,10 @@ def post(self, request):
                     availability_str,
                     # "type_name": item.type_name,
                 }
-                cwe = get_cve_from_cve_nums(cur_data["vul_cve_nums"])
-                if cwe:
-                    cur_data['vul_cve_nums']['cwe_num'] = cwe
+                if cur_data["vul_cve_nums"]:
+                    cwe = get_cve_from_cve_nums(cur_data["vul_cve_nums"])
+                    if cwe:
+                        cur_data['vul_cve_nums']['cwe_num'] = cwe
                 vul_ids.append(item.id)
                 content_list.append(cur_data)
             # 追加 用户 权限
@@ -318,6 +320,7 @@ def post(self, request):
             for row in content_list:
                 row["pro_info"] = pro_arr.get(row['id'], [])
                 row['type_name'] = ",".join(type_arr.get(row['id'], []))
+            set_vul_inetration(content_list, vul_ids, request.user.id)
         return R.success(data={
             'messages': content_list,
             'page': {
@@ -327,6 +330,13 @@ def post(self, request):
         }, )
 
 
+def set_vul_inetration(
+    content_list: list[dict[str, Any]],
+    vul_ids: list[int],
+    user_id: int,
+) -> None:
+    pass
+
 def get_vul_list_from_elastic_search(user_id,
                                      project_ids=[],
                                      project_version_ids=[],

dongtai_web/aggr_vul/app_vul_list.py

@@ -1,3 +1,4 @@
+from typing import Any
 from rest_framework.serializers import ValidationError
 from dongtai_common.endpoint import R
 from dongtai_common.endpoint import UserEndPoint
@@ -218,6 +219,7 @@ def post(self, request):
         for i in end['data']:
             i['status__name'] = status_obj.get(i['status_id'], "")
 
+        set_vul_inetration(end, request.user.id)
         return R.success(data={
             'messages': end['data'],
             'page': {
@@ -226,6 +228,9 @@ def post(self, request):
             }
         }, )
 
+def set_vul_inetration(end: dict[str, Any], user_id: int) -> None:
+    pass
+
 
 def get_vul_list_from_elastic_search(departments,
                                      project_ids=[],

dongtai_web/apitimelog/views.py

@@ -1,4 +1,4 @@
-from apitimelog.middleware import REQUEST_DICT
+from dongtai_web.apitimelog.middleware import REQUEST_DICT
 # Create your views here.
 
 from dongtai_common.endpoint import UserEndPoint