v1.15.0

.gitignore

@@ -32,3 +32,4 @@ celerybeat.pid
 *.o
 *.c
 *.prof
+*.dat

Pipfile

@@ -75,7 +75,7 @@ pillow = "==9.3.0"
 pyrsistent = "==0.19.1"
 pytz = "==2022.6"
 types-pyyaml = ">=6.0.12.2"
-uwsgi = "==2.0.21"
+uwsgi = "==2.0.22"
 marisa-trie = "==0.8.0"
 gunicorn = "==20.1.0"
 celery-singleton = "*"
@@ -89,6 +89,11 @@ networkit = "*"
 flower = "~=2.0.0"
 django-health-check = "==3.17.0"
 django-prometheus = "==2.3.1"
+django-add-default-value = "==0.10.0"
+networkx = {extras = ["all"], version = "*"}
+pandas = "~=2.1.0"
+pydot = "*"
+more-itertools = "*"
 
 [dev-packages]
 

deploy/commands/management/commands/load_hook_strategy.py

@@ -3,9 +3,11 @@
 from collections import OrderedDict
 
 from django.core.management.base import BaseCommand
+from django.db.models import Q
 
 from dongtai_common.models.hook_strategy import HookStrategy
 from dongtai_common.models.hook_type import HookType
+from dongtai_common.models.sensitive_info import IastSensitiveInfoRule
 from dongtai_common.models.strategy import IastStrategyModel
 from dongtai_common.utils.validate import save_hook_stratefile_sha1sum
 from dongtai_conf.settings import BASE_DIR
@@ -23,6 +25,9 @@ def handle(self, *args, **options):
         POLICY_DIR = os.path.join(BASE_DIR, "static/data/")
         with open(os.path.join(POLICY_DIR, "vul_strategy.json")) as fp:
             full_strategies = json.load(fp, object_pairs_hook=OrderedDict)
+        if os.path.exists(os.path.join(POLICY_DIR, "sensitive_info_strategy.json")):
+            with open(os.path.exists(os.path.join(POLICY_DIR, "sensitive_info_strategy.json"))) as fp:
+                full_strategies.extend(json.load(fp, object_pairs_hook=OrderedDict))
         strategy_dict = {}
         for strategy in full_strategies:
             if IastStrategyModel.objects.filter(
@@ -110,7 +115,6 @@ def handle(self, *args, **options):
                 hooktype_obj.save()
                 hooktype_dict[f"{hook_type['value']}-{hook_type['type']}"] = hooktype_obj
 
-            HookStrategy.objects.filter(language_id=v, system_type=1).delete()
             with open(os.path.join(POLICY_DIR, f"{k.lower()}_full_policy.json")) as fp:
                 full_policy = json.load(fp, object_pairs_hook=OrderedDict)
             for policy in full_policy:
@@ -119,6 +123,21 @@ def handle(self, *args, **options):
                         continue
                     policy_strategy = strategy_dict[policy["value"]]
                     for hook_strategy in policy["details"]:
+                        if HookStrategy.objects.filter(
+                            value=hook_strategy["value"], type=hook_strategy["type"], language_id=v, system_type=1
+                        ).exists():
+                            # 如果已经存在规则,跳过创建
+                            continue
+                        if HookStrategy.objects.filter(
+                            value=hook_strategy["value"], type=hook_strategy["type"], language_id=v, system_type=0
+                        ):
+                            # 如果已经存在用户自定义规则,设置为系统规则,跳过创建
+                            hook_strategy_obj = HookStrategy.objects.filter(
+                                value=hook_strategy["value"], type=hook_strategy["type"], language_id=v, system_type=0
+                            ).get()
+                            hook_strategy_obj.system_type = 1
+                            hook_strategy_obj.save()
+                            continue
                         del hook_strategy["language"]
                         hook_strategy["language_id"] = v
                         HookStrategy.objects.create(strategy=policy_strategy, **hook_strategy)
@@ -127,8 +146,49 @@ def handle(self, *args, **options):
                         continue
                     policy_hook_type = hooktype_dict[f"{policy['value']}-{policy['type']}"]
                     for hook_strategy in policy["details"]:
+                        if HookStrategy.objects.filter(
+                            value=hook_strategy["value"], type=hook_strategy["type"], language_id=v, system_type=1
+                        ).exists():
+                            # 如果已经存在规则,跳过创建
+                            continue
+                        if HookStrategy.objects.filter(
+                            value=hook_strategy["value"], type=hook_strategy["type"], language_id=v, system_type=0
+                        ):
+                            # 如果已经存在用户自定义规则,设置为系统规则,跳过创建
+                            hook_strategy_obj = HookStrategy.objects.filter(
+                                value=hook_strategy["value"], type=hook_strategy["type"], language_id=v, system_type=0
+                            ).get()
+                            hook_strategy_obj.system_type = 1
+                            hook_strategy_obj.save()
+                            continue
                         del hook_strategy["language"]
                         hook_strategy["language_id"] = v
                         HookStrategy.objects.create(hooktype=policy_hook_type, **hook_strategy)
         save_hook_stratefile_sha1sum()
+
+        sensitive_info_rule = []
+        if os.path.exists(os.path.join(POLICY_DIR, "sensitive_info_rule.json")):
+            with open(os.path.join(POLICY_DIR, "sensitive_info_rule.json")) as fp:
+                sensitive_info_rule = json.load(fp, object_pairs_hook=OrderedDict)
+        sensitive_info_rule_ids = []
+        for rule in sensitive_info_rule:
+            if rule["strategy"] not in strategy_dict:
+                continue
+            strategy = strategy_dict[rule["strategy"]]
+            exist_rule = IastSensitiveInfoRule.objects.filter(
+                strategy=strategy, pattern_type_id=rule["pattern_type"], pattern=rule["pattern"], system_type=1
+            ).first()
+            if exist_rule:
+                sensitive_info_rule_ids.append(exist_rule.pk)
+            else:
+                obj = IastSensitiveInfoRule.objects.create(
+                    user_id=1,
+                    strategy=strategy,
+                    pattern_type_id=rule["pattern_type"],
+                    pattern=rule["pattern"],
+                    status=1,
+                    system_type=1,
+                )
+                sensitive_info_rule_ids.append(obj.pk)
+        IastSensitiveInfoRule.objects.filter(~Q(id__in=sensitive_info_rule_ids), system_type=1).delete()
         self.stdout.write(self.style.SUCCESS("Successfully load strategy ."))

deploy/commands/management/commands/unlock_user.py

@@ -0,0 +1,16 @@
+from django.core.management.base import BaseCommand
+
+from dongtai_common.models.user import User
+
+
+class Command(BaseCommand):
+    help = "scripts to unlock user"
+    functions = []
+
+    def add_arguments(self, parser):
+        parser.add_argument("id", nargs="*", default=[], type=int)
+
+    def handle(self, *args, **options):
+        users = User.objects.filter(pk__in=options["id"]).all() if options["id"] else User.objects.all()
+        users.update(failed_login_count=0)
+        self.stdout.write(self.style.SUCCESS("Successfully Unlock Users"))

deploy/kubernetes/helm/templates/_helpers.tpl

@@ -150,7 +150,7 @@ initContainers:
 
 {{- define "deploy.initContainers" -}}
 initContainers:
-  - image: {{ .Values.images }}/dongtai-logrotate:{{ .Values.tag }}
+  - image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }}
     command:
     - sh
     - -c
@@ -247,7 +247,7 @@ Create the name of the service account to use
 
     [security]
     csrf_trust_origins ={{.Values.csrfTrustOrigins}}
-    secret_key ={{.Values.secretKey}}
+    secret_key ={{ randAlphaNum 50 }}
 
     [smtp]
     server ={{.Values.smtp.server}}
@@ -345,6 +345,17 @@ Create the name of the service account to use
             location /log/ {
              proxy_pass http://dongtai-logstash-svc:8082/;
             }
+            {{- if .Values.max }}
+            location /dongtai_doc/ {
+             proxy_pass http://dongtai-doc-svc/;
+             proxy_set_header X-Scheme $scheme;
+             proxy_set_header X-Forwarded-Proto $scheme;
+             proxy_set_header X-real-ip $remote_addr;
+             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+             proxy_set_header User-Agent $http_user_agent;
+             proxy_set_header X-Host $http_x_forwarded_host;
+            }
+            {{- end }}
             location = /50x.html {
                 root   /usr/share/nginx/html;
             }

deploy/kubernetes/helm/templates/deployments/dongtai-doc.yaml

@@ -0,0 +1,43 @@
+{{- if .Values.max }}
+---
+# dongtai-doc服务
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "dongtai.fullname" . }}-doc
+  namespace: {{.Release.Namespace}}
+  annotations:
+    kubesphere.io/description: {{ template "dongtai.fullname" . }}-doc
+  labels:
+    app: {{ template "dongtai.fullname" . }}-doc
+    {{- include "dongtai.labels" .  | nindent 4 }}
+spec:
+  replicas: {{.Values.replicaCount}}
+  selector:
+    matchLabels:
+      app: {{ template "dongtai.fullname" . }}-doc
+      {{- include "dongtai.labels" .  | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        doc_number: {{.Values.build.server_number}}
+      labels:
+        app: {{ template "dongtai.fullname" . }}-doc
+        {{- include "dongtai.labels" .  | nindent 8 }}
+    spec:
+{{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+{{- end }}
+      containers:
+        - name: {{ template "dongtai.fullname" . }}-doc-container
+          image: {{ .Values.images }}/dongtai-doc:{{ .Values.tag }}
+          imagePullPolicy: Always
+          resources:
+            limits:
+              cpu: 500m
+              memory: 500Mi
+            requests:
+              cpu: 500m
+              memory: 500Mi
+{{- end }}

deploy/kubernetes/helm/templates/job/dongtai_update.yaml

@@ -0,0 +1,38 @@
+{{- if .Values.migrate }}
+---
+# dongtai-update服务
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ template "dongtai.fullname" . }}-update-{{ randNumeric 10 }}
+  namespace: {{.Release.Namespace}}
+  annotations:
+    {{- if not .Values.develop.dev }}
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+    {{- end }}
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-5"
+    kubesphere.io/description: {{ template "dongtai.fullname" . }}-update
+  labels:
+    app: {{ template "dongtai.fullname" . }}-update
+    {{- include "dongtai.labels" .  | nindent 4 }}
+spec:
+  template:
+    metadata:
+      labels:
+        app: {{ template "dongtai.fullname" . }}-update
+        {{- include "dongtai.labels" .  | nindent 8 }}
+    spec:
+      restartPolicy: Never
+{{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+{{- end }}
+      containers:
+        - name: {{ template "dongtai.fullname" . }}-update-container
+          image: {{ .Values.images }}/dongtai-server:{{ .Values.tag }}
+          command: [ "/bin/sh","/opt/dongtai/deploy/docker/entrypoint.sh" ]
+          args: [ "migrate" ]
+          {{- include "deploy.config" . | nindent 10 }}
+      {{- include "deploy.config.vo" . | nindent 6 }}
+{{- end }}

deploy/kubernetes/helm/templates/service/dongtai-doc.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.max }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: dongtai-doc-svc
+  namespace: {{.Release.Namespace}}
+spec:
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 80
+  selector:
+    app: {{ template "dongtai.fullname" . }}-doc
+  type: ClusterIP
+{{- end }}

deploy/kubernetes/helm/values.yaml

@@ -17,6 +17,7 @@ logging_level: INFO # DEBUG, INFO
 somaxconn: null #If system max net.core.somaxconn (128) . Example: somaxconn: 4096
 healthcheck: true
 logstash: "true"
+migrate: true
 
 nodeSelector:
   kubernetes.io/os: linux
@@ -54,7 +55,6 @@ storage:
   persistentVolumeClaim: iast-agent-pvc  # or null(The agent needs to close the reporting log function 'dongtai.log.disable-collector: false')
 
 csrfTrustOrigins: .example.com
-secretKey: vbjlvbxfvazjfprywuxgyclmvhtmselddsefxxlcixovmqfpgy
 
 smtp:
   server: smtp_server

dongtai_common/common/utils/__init__.py

@@ -159,3 +159,38 @@ def authenticate(self, request):
             return None
         token = auth.lower().replace(self.keyword.lower().encode(), b"", 1).decode()
         return self.auth_decodedenticate_credentials(token)
+
+
+@cached_decorator(random_range=(300, 600), use_celery_update=False)
+def get_user_from_project_key(key):
+    from dongtai_common.models.project import IastProject
+    from dongtai_common.models.user import User
+
+    project = IastProject.objects.get(token=key)
+    principal = User.objects.filter(pk=project.user_id).first()
+    user = principal if principal else User.objects.filter(pk=1).first()
+    user.using_project = project
+    return user
+
+
+class ProjectTokenAuthentication(TokenAuthentication):
+    keyword = "Token PROJECT"
+    model = None
+
+    def auth_decodedenticate_credentials(self, key):
+        from rest_framework import exceptions
+
+        from dongtai_common.models.project import IastProject
+
+        try:
+            user = get_user_from_project_key(key)
+        except IastProject.DoesNotExist as e:
+            raise exceptions.AuthenticationFailed(_("Invalid token.")) from e
+        return (user, key)
+
+    def authenticate(self, request):
+        auth = get_authorization_header(request)
+        if not auth or not auth.lower().startswith(self.keyword.lower().encode()):
+            return None
+        token = auth[13:].decode()
+        return self.auth_decodedenticate_credentials(token)