Merge pull request #1636 from HXSecurity/beta

Beta
HXSecurity · Jul 17, 2023 · 852ade2 · 852ade2
2 parents c061d2d + 2494171
commit 852ade2
Show file tree

Hide file tree

Showing 266 changed files with 5,110 additions and 11,315 deletions.
diff --git a/.github/deploy/deploy-dongtai-server-test.yml b/.github/deploy/deploy-dongtai-server-test.yml
diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml
@@ -11,9 +11,6 @@ on:
       agent_version:
         required: true
         type: string
-      # server_version:
-      #   required: true
-      #   type: string
 
 jobs:
   Deploy-to-dev:
@@ -27,12 +24,6 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      # - name: Checkout
-      #   if: ${{ inputs.server_version }}
-      #   uses: actions/checkout@v3
-      #   with:
-      #     ref: ${{ inputs.server_version }}
-
       - name: Get the release version
         id: version
         run: echo ::set-output name=GITHUB_REF::${GITHUB_REF##*/}
@@ -63,7 +54,7 @@ jobs:
           access-key-secret: ${{ secrets.CHART_OSS_ACCESS_KEY_SECRET }}
 
       - name: Download Agent dev
-        if: ${{ steps.version.outputs.GITHUB_REF }} == develop || ${{ steps.version.outputs.GITHUB_REF }} == beta || ${{ steps.version.outputs.GITHUB_REF }} == main
+#        if: ${{ steps.version.outputs.GITHUB_REF }} == develop || ${{ steps.version.outputs.GITHUB_REF }} == beta || ${{ steps.version.outputs.GITHUB_REF }} == main
         run: |
           if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then ossutil cp oss://dongtai-helm-charts/agent_test/java/latest/ ./ --include "*.jar" -r
           else  ossutil cp oss://dongtai-helm-charts/agent_${{ steps.version.outputs.GITHUB_REF }}/java/latest/ ./ --include "*.jar" -r
@@ -86,18 +77,23 @@ jobs:
           push: true
           platforms: linux/amd64
           tags: | 
-            registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:latest-${{ steps.version.outputs.GITHUB_REF }}
-            registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server-test:1.0.${{github.run_number}}-${{ steps.version.outputs.GITHUB_REF }}
+            registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-latest
+            registry.cn-hongkong.aliyuncs.com/secnium/dongtai-server:${{ steps.version.outputs.GITHUB_REF }}-1.0.${{github.run_number}}
+
+      - name: Set the value
+        id: release
+        run: |
+          if [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo "helm_ns=test" >> $GITHUB_ENV; echo "helm_mysql=test" >> $GITHUB_ENV 
+          elif [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then echo "helm_ns=beta" >> $GITHUB_ENV; echo "helm_mysql=beta" >> $GITHUB_ENV 
+          else echo "helm_ns=main" >> $GITHUB_ENV ; echo "helm_mysql=temp" >> $GITHUB_ENV ;fi
 
-      - uses: actions/checkout@master
       - name: deploy to cluster
         uses: wahyd4/kubectl-helm-action@master
         env:
           KUBE_CONFIG_DATA: ${{ secrets.KUBE_CONFIG_TEST_DATA }}
         with:
           args: |
-            if [ ${{ steps.version.outputs.GITHUB_REF }} = beta ] ; then find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/iast-test/iast-beta/g' {} \; 
-            elif [ ${{ steps.version.outputs.GITHUB_REF }} = develop ] ; then echo testing !!!
-            else find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/iast-test/iast-main/g' {} \; ;fi
-            find .github/deploy/deploy-dongtai-server-test.yml -type f -exec sed -i 's/VERSION/1.0.${{github.run_number}}-${{ steps.version.outputs.GITHUB_REF }}/g' {} \;
-            ls .github/deploy/deploy-dongtai-server-test.yml | xargs -I {} kubectl apply -f {}
+            git clone https://github.com/HXSecurity/DongTai.git
+            helm upgrade --install huoxian --create-namespace -n iast-${{ env.helm_ns }} ./DongTai/deploy/kubernetes/helm/ \
+            --set sca.sca_token=${{ secrets.TOKEN_SCA }} --set usb.usb_token=${{ secrets.TOKEN_SCA }} --set mysql.host=iast-mysql-${{ env.helm_mysql }}.huoxian.cn \
+            --set tag=${{ steps.version.outputs.GITHUB_REF }}-latest --set build.server_number=iast${{github.run_number}} --set develop.agentZip=${{ env.helm_ns }} --values https://charts.dongtai.io/devops.yaml
diff --git a/.github/workflows/teststate.yml b/.github/workflows/teststate.yml
@@ -124,7 +124,7 @@ jobs:
           
       - name: Django Unit Testing
         run: |
-          mypy --show-error-codes --ignore-missing-imports  --no-incremental --show-error-codes --check-untyped-defs --disable-error-code var-annotated  --disable-error-code  list-item  --disable-error-code attr-defined --disable-error-code arg-type --disable-error-code assignment --disable-error-code misc --disable-error-code union-attr --disable-error-code index --disable-error-code call-overload  --disable-error-code dict-item  --disable-error-code truthy-function --disable-error-code operator --disable-error-code name-defined .
+          mypy --show-error-codes --ignore-missing-imports  --no-incremental --show-error-codes --check-untyped-defs --disable-error-code var-annotated  --disable-error-code  list-item  --disable-error-code attr-defined --disable-error-code assignment --disable-error-code misc --disable-error-code union-attr --disable-error-code index --disable-error-code call-overload  --disable-error-code dict-item  --disable-error-code truthy-function --disable-error-code operator --disable-error-code name-defined .
 
   Run-Pep8Check:
     runs-on: ubuntu-latest
@@ -244,6 +244,7 @@ jobs:
           
       - name: Django Unit Testing
         run: |
+          export CELERY_EAGER_TEST=TRUE
           cp dongtai_conf/conf/config.ini.test dongtai_conf/conf/config.ini
           mkdir -p /tmp/logstash/report/{img,word,pdf,excel} && mkdir -p /tmp/iast_cache/package && mkdir -p /tmp/logstash/batchagent 
           python3 ./deploy/docker/version_update.py || true
@@ -293,6 +294,7 @@ jobs:
           
       - name: Django Unit Testing
         run: |
+          export CELERY_EAGER_TEST=TRUE
           cp dongtai_conf/conf/config.ini.test dongtai_conf/conf/config.ini
           mkdir -p /tmp/logstash/report/{img,word,pdf,excel} && mkdir -p /tmp/iast_cache/package && mkdir -p /tmp/logstash/batchagent 
           python3 ./deploy/docker/version_update.py || true

diff --git a/Pipfile b/Pipfile
@@ -69,7 +69,7 @@ botocore = "==1.27.91"
 setuptools = "==65.5.1"
 elasticsearch = "==7.17.7"
 django-mock-queries = "==v2.1.7"
-cryptography = "==41.0.0"
+cryptography = "==41.0.2"
 jsonschema = "==4.17.0"
 pillow = "==9.3.0"
 pyrsistent = "==0.19.1"
@@ -84,6 +84,11 @@ django-seriously = "*"
 dataclasses-json = "*"
 django-silk = "*"
 types-python-dateutil = "*"
+shortuuid = "*"
+networkit = "*"
+flower = "~=2.0.0"
+django-health-check = "==3.17.0"
+django-prometheus = "==2.3.1"
 
 [dev-packages]