add visibility plugin

sermant-io · Jan 9, 2023 · af5cf69 · af5cf69
1 parent 315f7d0
commit af5cf69
Show file tree

Hide file tree

Showing 90 changed files with 5,153 additions and 40 deletions.
diff --git a/.github/actions/common/plugin-change-check/action.yml b/.github/actions/common/plugin-change-check/action.yml
@@ -111,6 +111,11 @@ runs:
             ${{ env.sermantMonitorChanged }} == 'true' ];then
           echo "enableMonitor=true" >> $GITHUB_ENV
         fi
+        # ==========visibility is needed to test?==========
+        if [ ${{ env.sermantAgentCoreChanged }} == 'true' -o \
+            ${{ env.sermantServiceVisibilityChanged }} == 'true' ];then
+          echo "enableVisibility=true" >> $GITHUB_ENV
+        fi
         # *****************spring_integration_test_1.yml*****************
         # ==========graceful is needed to test?==========
         if [ ${{ env.sermantAgentCoreChanged }} == 'true' -o ${{ env.sermantServiceRegistryChanged }} == 'true' ];then
@@ -150,5 +155,6 @@ runs:
           echo "enableGraceful=true" >> $GITHUB_ENV
           echo "enableMonitor=true" >> $GITHUB_ENV
           echo "enableSpringCommon=true" >> $GITHUB_ENV
+          echo "enableVisibility=true" >> $GITHUB_ENV
         fi
 
diff --git a/.github/actions/scenarios/dubbo/visibility/action.yml b/.github/actions/scenarios/dubbo/visibility/action.yml
@@ -0,0 +1,73 @@
+name: "Visibility Test"
+description: "Auto test for plugin sermant-visibility"
+runs:
+  using: "composite"
+  steps:
+    - name: visibility-entry
+      uses: ./.github/actions/common/entry
+      with:
+        log-dir: ./logs/dubbo-visibility
+    - name: package visibility dubbo 2.6.0 tests
+      shell: bash
+      if: matrix.dubbo-version == '2-6' && matrix.dubbo-versions == '0'
+      run: mvn package -Dalibaba.dubbo.version=2.6.${{ matrix.dubbo-versions }} -DskipTests -P260 --file sermant-integration-tests/dubbo-test/pom.xml
+    - name: package visibility dubbo 2.6.x tests
+      shell: bash
+      if: matrix.dubbo-version == '2-6' && matrix.dubbo-versions != '0'
+      run: mvn package -Dalibaba.dubbo.version=2.6.${{ matrix.dubbo-versions }} -DskipTests --file sermant-integration-tests/dubbo-test/pom.xml
+    - name: package visibility dubbo 2.7.x tests
+      shell: bash
+      if: matrix.dubbo-version == '2-7'
+      run: mvn package -Ddubbo.version=2.7.${{ matrix.dubbo-versions }} -DskipTests --file sermant-integration-tests/dubbo-test/pom.xml
+    - name: start visibility provider service
+      shell: bash
+      env:
+        SERVER_PORT: 28054
+        DUBBO_PROTOCOL_PORT: 28854
+        VISIBILITY_SERVICE_FLAG: true
+        VISIBILITY_CONFIG_START_FLAG: true
+        AGENT_CONFIG_SERVICE_BLACK_LIST: com.huaweicloud.sermant.implement.service.tracing.TracingServiceImpl
+      run: |
+        nohup java -javaagent:sermant-agent-${{ env.sermantVersion }}/agent/sermant-agent.jar=appName=dubbo-integration-provider  -jar \
+        sermant-integration-tests/dubbo-test/dubbo-${{ matrix.dubbo-version }}-integration-provider/target/dubbo-integration-provider.jar > ${{ env.logDir }}/visibility-provider.log 2>&1 &
+    - name: start visibility consumer service
+      shell: bash
+      env:
+        SERVER_PORT: 28050
+        DUBBO_PROTOCOL_PORT: 28850
+        VISIBILITY_SERVICE_FLAG: true
+        VISIBILITY_CONFIG_START_FLAG: true
+        AGENT_CONFIG_SERVICE_BLACK_LIST: com.huaweicloud.sermant.implement.service.tracing.TracingServiceImpl
+      run: |
+        nohup java -javaagent:sermant-agent-${{ env.sermantVersion }}/agent/sermant-agent.jar=appName=dubbo-integration-consumer -jar \
+        sermant-integration-tests/dubbo-test/dubbo-${{ matrix.dubbo-version }}-integration-consumer/target/dubbo-integration-consumer.jar > ${{ env.logDir }}/visibility-consumer.log 2>&1 &
+    - name: start visibility backend service
+      shell: bash
+      run: |
+        nohup java -jar sermant-agent-${{ env.sermantVersion }}/server/sermant-backend-1.0.0.jar > ${{ env.logDir }}/visibility-backend.log 2>&1 &
+    - name: waiting for visibility services start
+      shell: bash
+      run: |
+        bash ./sermant-integration-tests/scripts/checkService.sh http://127.0.0.1:28050/actuator/health 120
+        bash ./sermant-integration-tests/scripts/checkService.sh http://127.0.0.1:28054/actuator/health 120
+        bash ./sermant-integration-tests/scripts/checkService.sh http://127.0.0.1:8900/visibility/getCollectorInfo 120
+    - name: visibility test
+      shell: bash
+      env:
+        TEST_TYPE: visibility
+      run: mvn  test --file sermant-integration-tests/dubbo-test/pom.xml
+    - name: exit
+      if: always()
+      uses: ./.github/actions/common/exit
+      with:
+        processor-keyword: dubbo
+    - name: upload error log
+      uses: actions/upload-artifact@v3
+      if: ${{ failure() || cancelled() }}
+      with:
+        name: visibility-(${{ matrix.dubbo-version }}-${{ matrix.dubbo-versions }})-logs
+        path: |
+          ./*.log
+          ./logs/**/*.log
+        if-no-files-found: warn
+        retention-days: 2
diff --git a/.github/actions/scenarios/spring/visibility/action.yml b/.github/actions/scenarios/spring/visibility/action.yml
@@ -0,0 +1,71 @@
+name: "Visibility Test For Spring"
+description: "Auto test for visibility"
+runs:
+  using: "composite"
+  steps:
+    - name: entry
+      uses: ./.github/actions/common/entry
+      with:
+        log-dir: ./logs/visibility
+    - name: 1.5.x config
+      if: matrix.springBootVersion == '1.5.0.RELEASE' && matrix.springCloudVersion == 'Edgware.SR2'
+      shell: bash
+      run: |
+        echo "tailVersion=-1.5.x" >> $GITHUB_ENV
+    - name: package common demos
+      shell: bash
+      run: |
+        sed -i 's|<version>${{ env.projectSpringBootVersion }}</version>|<version>${{ matrix.springBootVersion }}</version>|g' sermant-integration-tests/spring-test/pom.xml
+        mvn package -Dspring.cloud.version=${{ matrix.springCloudVersion }} -Dspring.boot.version=${{ matrix.springBootVersion }} -DskipTests -P common-test${{ env.tailVersion }} --file sermant-integration-tests/spring-test/pom.xml
+    - name: start visibility provider applications
+      shell: bash
+      env:
+        VISIBILITY_SERVICE_FLAG: true
+        VISIBILITY_CONFIG_START_FLAG: true
+        AGENT_CONFIG_SERVICE_BLACK_LIST: com.huaweicloud.sermant.implement.service.tracing.TracingServiceImpl
+      run: |
+        nohup java -javaagent:sermant-agent-${{ env.sermantVersion }}/agent/sermant-agent.jar=appName=default -Dservice.meta.application=feign -Dserver.port=8013 -jar \
+        -Dsermant_log_dir=${{ env.logDir }}/feign-provider${{ env.tailVersion }} -Dconfig.retry.sleepMs=20 \
+        sermant-integration-tests/spring-test/spring-common-demos/spring-common-feign${{ env.tailVersion }}/feign-provider${{ env.tailVersion }}/target/feign-provider${{ env.tailVersion }}.jar > ${{ env.logDir }}/visibility-feign-provider.log 2>&1 &
+        nohup java -javaagent:sermant-agent-${{ env.sermantVersion }}/agent/sermant-agent.jar=appName=default -Dservice.meta.application=feign -Dserver.port=8014 \
+        -Dsermant_log_dir=${{ env.logDir }}/feign-provider-8014${{ env.tailVersion }} -Dservicecomb.service.enableSpringRegister=true -jar \
+        sermant-integration-tests/spring-test/spring-common-demos/spring-common-feign${{ env.tailVersion }}/feign-provider${{ env.tailVersion }}/target/feign-provider${{ env.tailVersion }}.jar > ${{ env.logDir }}/visibility-feign-provider-8014.log 2>&1 &
+    - name: start visibility consumer applications
+      shell: bash
+      env:
+        VISIBILITY_SERVICE_FLAG: true
+        VISIBILITY_CONFIG_START_FLAG: true
+        AGENT_CONFIG_SERVICE_BLACK_LIST: com.huaweicloud.sermant.implement.service.tracing.TracingServiceImpl
+      run: |
+        nohup java -javaagent:sermant-agent-${{ env.sermantVersion }}/agent/sermant-agent.jar=appName=default -Dservice.meta.application=feign -Dserver.port=8015 -jar \
+                -Dsermant_log_dir=${{ env.logDir }}/feign-consumer${{ env.tailVersion }} -Dservicecomb.service.openMigration=true -Dservicecomb.service.enableSpringRegister=true \
+                sermant-integration-tests/spring-test/spring-common-demos/spring-common-feign${{ env.tailVersion }}/feign-consumer${{ env.tailVersion }}/target/feign-consumer${{ env.tailVersion }}.jar > ${{ env.logDir }}/visibility-feign-consumer.log 2>&1 &
+    - name: start visibility backend service
+      shell: bash
+      run: |
+        nohup java -jar sermant-agent-${{ env.sermantVersion }}/server/sermant-backend-1.0.0.jar > ${{ env.logDir }}/visibility-backend.log 2>&1 &
+    - name: waiting for services start
+      shell: bash
+      run: |
+        ps -ef | grep java
+        bash ./sermant-integration-tests/scripts/checkService.sh http://127.0.0.1:8015/flowcontrol/rateLimiting 120
+        bash ./sermant-integration-tests/scripts/checkService.sh http://127.0.0.1:8014/rateLimiting 60
+        bash ./sermant-integration-tests/scripts/checkService.sh http://127.0.0.1:8900/visibility/getCollectorInfo 120
+    - name: integration test module visibility
+      shell: bash
+      run: mvn test -Dsermant.integration.test.type=VISIBILITY --file sermant-integration-tests/spring-test/pom.xml
+    - name: exit
+      if: always()
+      uses: ./.github/actions/common/exit
+      with:
+        processor-keyword: feign|rest
+    - name: if failure then upload error log
+      uses: actions/upload-artifact@v3
+      if: ${{ failure() || cancelled() }}
+      with:
+        name: (${{ github.job }})-visibility-(${{ matrix.springBootVersion }}-${{ matrix.springCloudVersion }})-logs
+        path: |
+          ./*.log
+          ./logs/**
+        if-no-files-found: warn
+        retention-days: 2
diff --git a/.github/workflows/dubbo_integration_test.yml b/.github/workflows/dubbo_integration_test.yml
@@ -17,6 +17,7 @@ on:
       - 'sermant-plugins/sermant-service-registry/**'
       - 'sermant-plugins/sermant-springboot-registry/**'
       - 'sermant-plugins/sermant-monitor/**'
+      - 'sermant-plugins/sermant-service-visibility/**'
       - '.github/workflows/dubbo_integration*.yaml'
       - '.github/actions/**'
 concurrency:
@@ -57,9 +58,12 @@ jobs:
         with:
           path: sermant-agent-*/
           key: ${{ runner.os }}-agent-${{ github.run_id }}
-      - name: package agent
+      - name: not package backend
+        if: env.enableVisibility != 'true'
         run: |
           sed -i '/sermant-backend/d' pom.xml
+      - name: package agent
+        run: |
           sed -i '/sermant-backend-lite/d' pom.xml
           sed -i '/sermant-integration-tests/d' pom.xml
           sed -i '/sermant-injector/d' pom.xml
@@ -107,3 +111,6 @@ jobs:
       - name: (dubbo common) test for ${{ matrix.dubbo-version }}
         if: env.enableDubboCommon == 'true'
         uses: ./.github/actions/scenarios/dubbo/dubbo-common
+      - name: visibility test for ${{ matrix.dubbo-version }}
+        if: env.enableVisibility == 'true'
+        uses: ./.github/actions/scenarios/dubbo/visibility
diff --git a/.github/workflows/spring_integration_test_1.yml b/.github/workflows/spring_integration_test_1.yml
@@ -77,9 +77,12 @@ jobs:
         with:
           path: sermant-agent-*/
           key: ${{ runner.os }}-agent-${{ github.run_id }}
-      - name: package agent
+      - name: not package backend
+        if: env.enableVisibility != 'true'
         run: |
           sed -i '/sermant-backend/d' pom.xml
+      - name: package agent
+        run: |
           sed -i '/sermant-backend-lite/d' pom.xml
           sed -i '/sermant-integration-tests/d' pom.xml
           sed -i '/sermant-injector/d' pom.xml
@@ -143,3 +146,6 @@ jobs:
       - name: monitor test for springboot=${{ matrix.springBootVersion }} springCloudVersion=${{ matrix.springCloudVersion }}
         if: env.enableMonitor == 'true'
         uses: ./.github/actions/scenarios/spring/monitor
+      - name: visibility test for springboot=${{ matrix.springBootVersion }} springCloudVersion=${{ matrix.springCloudVersion }}
+        if: env.enableVisibility == 'true' == 'true'
+        uses: ./.github/actions/scenarios/spring/visibility
diff --git a/sermant-agentcore/sermant-agentcore-config/config/config.properties b/sermant-agentcore/sermant-agentcore-config/config/config.properties
@@ -36,3 +36,7 @@ service.meta.version=1.0.0
 service.meta.project=default
 service.meta.environment=
 service.meta.zone=
+
+visibility.service.groupName= 111
+visibility.service.version= 12345
+visibility.service.flag=false
diff --git a/sermant-agentcore/sermant-agentcore-config/config/plugins.yaml b/sermant-agentcore/sermant-agentcore-config/config/plugins.yaml
@@ -8,6 +8,7 @@ plugins:
   - monitor
   - springboot-registry
   - mq-consume-deny
+  - service-visibility
 profiles:
   cse:
     - flowcontrol

diff --git a/.../src/main/java/com/huaweicloud/sermant/core/service/visibility/api/VisibilityService.java b/.../src/main/java/com/huaweicloud/sermant/core/service/visibility/api/VisibilityService.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2022-2022 Huawei Technologies Co., Ltd. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.huaweicloud.sermant.core.service.visibility.api;
+
+import com.huaweicloud.sermant.core.service.BaseService;
+
+/**
+ * 服务下线接口处理
+ *
+ * @author zhp
+ * @since 2022-12-10
+ */
+public interface VisibilityService extends BaseService {
+    /**
+     * 服务重连处理
+     */
+    void reconnectHandler();
+}
diff --git a/.../src/main/java/com/huaweicloud/sermant/core/service/visibility/common/CollectorCache.java b/.../src/main/java/com/huaweicloud/sermant/core/service/visibility/common/CollectorCache.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright (C) 2021-2022 Huawei Technologies Co., Ltd. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.huaweicloud.sermant.core.service.visibility.common;
+
+import com.huaweicloud.sermant.core.service.visibility.entity.Consanguinity;
+import com.huaweicloud.sermant.core.service.visibility.entity.Contract;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * 采集信息缓存类
+ *
+ * @author zhp
+ * @since 2022-12-02
+ */
+public class CollectorCache {
+    /**
+     * 契约信息
+     */
+    public static final Map<String, Contract> CONTRACT_MAP = new ConcurrentHashMap<>();
+
+    /**
+     * 血缘关系
+     */
+    public static final Map<String, Consanguinity> CONSANGUINITY_MAP = new ConcurrentHashMap<>();
+
+    private CollectorCache() {
+    }
+
+    /**
+     * 保存契约信息
+     *
+     * @param contract 契约信息
+     */
+    public static void saveContractInfo(Contract contract) {
+        if (!CONTRACT_MAP.containsKey(contract.getServiceKey())) {
+            CONTRACT_MAP.putIfAbsent(contract.getServiceKey(), contract);
+        } else if (Objects.equals(contract.getType(), ServiceType.DUBBO.getName())) {
+            Contract oldContract = CONTRACT_MAP.get(contract.getServiceKey());
+            oldContract.setMethodInfoList(contract.getMethodInfoList());
+        } else {
+            Contract oldContract = CONTRACT_MAP.get(contract.getServiceKey());
+            oldContract.getMethodInfoList().addAll(contract.getMethodInfoList());
+        }
+    }
+
+    /**
+     * 保存血缘关系信息
+     *
+     * @param consanguinity 血缘关系信息
+     */
+    public static void saveConsanguinity(Consanguinity consanguinity) {
+        if (CONSANGUINITY_MAP.containsKey(consanguinity.getServiceKey())) {
+            Consanguinity consanguinityOld = CONSANGUINITY_MAP.get(consanguinity.getInterfaceName());
+            consanguinityOld.setProviders(consanguinity.getProviders());
+        } else {
+            CONSANGUINITY_MAP.putIfAbsent(consanguinity.getServiceKey(), consanguinity);
+        }
+    }
+}