xDS Service Discovery Integration Test

Signed-off-by: daizhenyu <[email protected]>

.github/actions/common/xds-service/action.yml

@@ -0,0 +1,51 @@
+name: "xDS Service Common Test"
+description: "Auto common test for xds service"
+runs:
+  using: composite
+  steps:
+    - name: start minikube
+      uses: medyagh/setup-minikube@latest
+      id: minikube
+      with:
+        minikube-version: 1.31.2
+        driver: docker
+        container-runtime: docker
+    - name: Delay for 10 seconds to ready k8s
+      shell: bash
+      run: sleep 10s
+    - name: check k8s
+      shell: bash
+      run: |
+        kubectl cluster-info
+        kubectl get pods -n kube-system
+        echo "k8s version:" $(kubectl version)
+    - name: deploy istio
+      shell: bash
+      run: |
+        curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.21.4 sh -
+        istio-1.21.4/bin/istioctl install -f sermant-integration-tests/xds-service-test/script/istio-operator.yaml --skip-confirmation
+    - name: Delay for 10 seconds to ready istio
+      shell: bash
+      run: |
+        sleep 10s
+        kubectl get pods -n istio-system
+    - name: Set up JDK ${{ env.javaVersion }}
+      uses: actions/setup-java@v3
+      with:
+        java-version: ${{ env.javaVersion }}
+        distribution: 'adopt'
+        cache: maven
+    - name: get cached agent
+      uses: actions/cache@v3
+      with:
+        path: sermant-agent-*/
+        key: ${{ runner.os }}-agent-${{ github.run_id }}
+    - name: remove xds service product
+      shell: bash
+      run: |
+        rm -rf sermant-integration-tests/xds-service-test/product
+    - name: get cached xds service package
+      uses: actions/cache@v3
+      with:
+        path: sermant-integration-tests/xds-service-test/product/
+        key: ${{ runner.os }}-xds-service-product-${{ github.run_id }}

.github/actions/scenarios/xds-service/xds-service-discovery/client-envoy/action.yml

@@ -0,0 +1,59 @@
+name: "xDS Service Discovery Test with Client Using Envoy"
+description: "Auto test for xds service discovery with Client Using Envoy"
+runs:
+  using: composite
+  steps:
+    - name: build docker image
+      shell: bash
+      run: |
+        cd sermant-integration-tests/xds-service-test/product/spring-server/
+        minikube image build -t spring-server:1.0.0 .
+        cd ../spring-client/
+        minikube image build -t spring-client:1.0.0 .
+        eval $(minikube docker-env)
+        docker images
+    - name: start spring-client using envoy
+      shell: bash
+      run: |
+        kubectl label namespace default istio-injection=enabled
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-client-sermant-xds-close.yaml
+        kubectl wait --for=condition=ready pod -l app=spring-client --timeout=10s
+        kubectl port-forward svc/spring-client 8080:8080 &
+        kubectl label namespace default istio-injection-
+    - name: Wait for spring-client pod to be ready
+      shell: bash
+      run: |
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-server-sermant-xds-open.yaml
+        kubectl wait --for=condition=ready pod -l app=spring-server --timeout=10s
+        sleep 5s
+    - name: test one server
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_ONE_SERVER_INSTANCE_ENVOY --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: stop spring-server
+      shell: bash
+      run: |
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-server-sermant-xds-open.yaml
+        sleep 3s
+    - name: test the number of spring-server instances changes from 1 to 0
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_ZERO_SERVER_INSTANCE_ENVOY --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: restart spring-server
+      shell: bash
+      run: |
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-server-sermant-xds-open.yaml
+        kubectl wait --for=condition=ready pod -l app=spring-server --timeout=10s
+        sleep 10s
+    - name: test the number of spring-server instances changes from 0 to 1
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_ONE_SERVER_INSTANCE_ENVOY --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: close all service
+      shell: bash
+      run: |
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-client-sermant-xds-close.yaml
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-server-sermant-xds-open.yaml

.github/actions/scenarios/xds-service/xds-service-discovery/sermant-only/action.yml

@@ -0,0 +1,84 @@
+name: "xDS Service Discovery Test"
+description: "Auto test for xds service discovery"
+runs:
+  using: composite
+  steps:
+    - name: build docker image
+      shell: bash
+      run: |
+        cd sermant-integration-tests/xds-service-test/product/spring-server/
+        minikube image build -t spring-server:1.0.0 .
+        cd ../spring-client/
+        minikube image build -t spring-client:1.0.0 .
+        eval $(minikube docker-env)
+        docker images
+    - name: start spring-client and spring-server
+      shell: bash
+      run: |
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-client-sermant-xds-open.yaml
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-server.yaml
+    - name: Wait for spring-client pod to be ready
+      shell: bash
+      run: |
+        kubectl wait --for=condition=ready pod -l app=spring-client --timeout=10s
+        kubectl wait --for=condition=ready pod -l app=spring-server --timeout=10s
+        kubectl port-forward svc/spring-client 8080:8080 &
+    - name: test one server
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_ONE_SERVER_INSTANCE --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: stop spring-server
+      shell: bash
+      run: |
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-server.yaml
+        sleep 3s
+    - name: test the number of spring-server instances changes from 1 to 0
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_ZERO_SERVER_INSTANCE --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: start spring-server with 2 replicas
+      shell: bash
+      run: |
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-server-2-replicas.yaml
+        kubectl wait --for=condition=ready pod -l app=spring-server --timeout=10s
+        sleep 2s
+    - name: test the number of spring-server instances changes from 0 to 2
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_TWO_SERVER_INSTANCE --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: create istio token
+      shell: bash
+      run: |
+        kubectl create serviceaccount istio-test
+        TOKEN=$(kubectl create token istio-test --duration=1h --audience=istio-ca)
+        kubectl create secret generic istio-test-secret \
+          --from-literal=token=${TOKEN} \
+          --type="istio.io/key-and-cert" \
+          -n default
+    - name: restart spring-client and sermant xds service use security mode
+      shell: bash
+      run: |
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-client-sermant-xds-open.yaml
+        sleep 5s
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-client-sermant-xds-subscribe-secure.yaml
+        kubectl wait --for=condition=ready pod -l app=spring-client --timeout=10s
+        pkill -f "kubectl port-forward svc/spring-client"
+        kubectl port-forward svc/spring-client 8080:8080 &
+    - name: test sermant xds service use security mode
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=SECRET --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: test subscribe to get service instance
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_SUBSCRIBE --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: close all service
+      shell: bash
+      run: |
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-client-sermant-xds-subscribe-secure.yaml
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-server-2-replicas.yaml

.github/actions/scenarios/xds-service/xds-service-discovery/server-envoy/action.yml

@@ -0,0 +1,58 @@
+name: "xDS Service Discovery Test with Server Using Envoy"
+description: "Auto test for xds service discovery with server using envoy"
+runs:
+  using: composite
+  steps:
+    - name: build docker image
+      shell: bash
+      run: |
+        cd sermant-integration-tests/xds-service-test/product/spring-server/
+        minikube image build -t spring-server:1.0.0 .
+        cd ../spring-client/
+        minikube image build -t spring-client:1.0.0 .
+        eval $(minikube docker-env)
+        docker images
+    - name: start spring-client
+      shell: bash
+      run: |
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-client-sermant-xds-open.yaml
+        kubectl wait --for=condition=ready pod -l app=spring-client --timeout=10s
+        kubectl port-forward svc/spring-client 8080:8080 &
+    - name: start spring-server with envoy
+      shell: bash
+      run: |
+        kubectl label namespace default istio-injection=enabled
+        sleep 5s
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-server.yaml
+        kubectl wait --for=condition=ready pod -l app=spring-server --timeout=10s
+    - name: test one server
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_ONE_SERVER_INSTANCE --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: stop spring-server
+      shell: bash
+      run: |
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-server.yaml
+        sleep 3s
+    - name: test the number of spring-server instances changes from 1 to 0
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_ZERO_SERVER_INSTANCE --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: start spring-server with 2 replicas
+      shell: bash
+      run: |
+        kubectl apply -f sermant-integration-tests/xds-service-test/script/discovery/spring-server-2-replicas.yaml
+        kubectl wait --for=condition=ready pod -l app=spring-server --timeout=10s
+        sleep 2s
+    - name: test the number of spring-server instances changes from 0 to 2
+      shell: bash
+      run: |
+        mvn test -Dxds.service.integration.test.type=DISCOVERY_TWO_SERVER_INSTANCE --file \
+        sermant-integration-tests/xds-service-test/xds-service-integration-test/pom.xml
+    - name: close all service
+      shell: bash
+      run: |
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-client-sermant-xds-open.yaml
+        kubectl delete -f sermant-integration-tests/xds-service-test/script/discovery/spring-server-2-replicas.yaml

.github/workflows/xds_service_test.yml

@@ -0,0 +1,88 @@
+name: xds integration test
+on:
+  push:
+  pull_request:
+    branches:
+      - '*'
+    paths:
+      - 'sermant-agentcore/**'
+      - 'sermant-integration-tests/xds-service-test/**'
+      - '.github/workflows/xds_service_test.yml'
+      - '.github/actions/common/xds-service/**'
+      - '.github/actions/scenarios/xds-service/**'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.head_ref }}
+  cancel-in-progress: true
+jobs:
+  build-agent-and-cache:
+    name: build agent and cache
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'adopt'
+          cache: maven
+      - name: cache agent
+        uses: actions/cache@v3
+        with:
+          path: sermant-agent-*/
+          key: ${{ runner.os }}-agent-${{ github.run_id }}
+      - name: cache xds service package
+        uses: actions/cache@v3
+        with:
+          path: sermant-integration-tests/xds-service-test/product/
+          key: ${{ runner.os }}-xds-service-product-${{ github.run_id }}
+      - name: package agent
+        run: |
+          sed -i '/sermant-backend/d' pom.xml
+          sed -i '/sermant-injector/d' pom.xml
+          mvn package -DskipTests -Ptest --file pom.xml
+          cp -r sermant-integration-tests/xds-service-test/product/spring-client/agent sermant-integration-tests/xds-service-test/product/spring-server/
+  test-for-xds-service-discovery-onlysermant:
+    name: Test for xds service discovery with only sermant
+    runs-on: ubuntu-latest
+    needs: [build-agent-and-cache]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 100
+      - name: set java version to environment
+        run: |
+          echo "javaVersion=8" >> $GITHUB_ENV
+      - name: xds common operation
+        uses: ./.github/actions/common/xds-service
+      - name: xds service discovery
+        uses: ./.github/actions/scenarios/xds-service/xds-service-discovery/sermant-only
+  test-for-xds-service-discovery-with-server-envoy:
+    name: Test for xds service discovery with spring-server using envoy
+    runs-on: ubuntu-latest
+    needs: [build-agent-and-cache]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 100
+      - name: set java version to environment
+        run: |
+          echo "javaVersion=8" >> $GITHUB_ENV
+      - name: xds common operation
+        uses: ./.github/actions/common/xds-service
+      - name: xds service discovery
+        uses: ./.github/actions/scenarios/xds-service/xds-service-discovery/server-envoy
+  test-for-xds-service-discovery-with-client-envoy:
+    name: Test for xds service discovery with spring-client using enovy
+    runs-on: ubuntu-latest
+    needs: [build-agent-and-cache]
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 100
+      - name: set java version to environment
+        run: |
+          echo "javaVersion=8" >> $GITHUB_ENV
+      - name: xds common operation
+        uses: ./.github/actions/common/xds-service
+      - name: xds service discovery
+        uses: ./.github/actions/scenarios/xds-service/xds-service-discovery/client-envoy

sermant-agentcore/sermant-agentcore-config/config/test/config.properties

@@ -34,6 +34,8 @@ agent.service.inject.enable=true
 agent.service.dynamic.config.enable=true
 # HTTP server switch
 agent.service.httpserver.enable=false
+# xDS service switch
+agent.service.xds.service.enable=false
 #============================= Event configuration =============================#
 # Event switch
 event.enable=false
@@ -91,6 +93,13 @@ gateway.nettyPort=6888
 #gateway.initReconnectInternalTime=5
 # Specify retreat algorithm maximum connection interval (s)
 #gateway.maxReconnectInternalTime=180
+#=============================xds configuration===============================#
+# istiod control plane address, security.enable=false with 15010 port, and security.enable=true with 15012 port
+xds.config.control.plane.address=istiod.istio-system.svc:15010
+# Whether to use secure communication with the control plane
+xds.config.security.enable=false
+# service account token used for secure communication with the control plane
+xds.config.service.account.token.path=/var/run/secrets/kubernetes.io/serviceaccount/token
 #=============================Metadata===============================#
 # Service name for host service instance
 service.meta.service=default

sermant-integration-tests/pom.xml

@@ -1,19 +1,4 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<!--
-  ~ Copyright (C) 2022-2022 Huawei Technologies Co., Ltd. All rights reserved.
-  ~
-  ~ Licensed under the Apache License, Version 2.0 (the "License");
-  ~ you may not use this file except in compliance with the License.
-  ~ You may obtain a copy of the License at
-  ~
-  ~     http://www.apache.org/licenses/LICENSE-2.0
-  ~
-  ~ Unless required by applicable law or agreed to in writing, software
-  ~ distributed under the License is distributed on an "AS IS" BASIS,
-  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  ~ See the License for the specific language governing permissions and
-  ~ limitations under the License.
-  -->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
@@ -35,5 +20,6 @@
         <module>tag-transmission-test</module>
         <module>mq-consume-prohibition-test</module>
         <module>database-write-prohibition-test</module>
+        <module>xds-service-test</module>
     </modules>
 </project>