Merge branch 'main' into feature/egress-gateway/add-egress-query

vdaas · Oct 16, 2023 · 1bd59a0 · 1bd59a0
2 parents 84858ab + b4028e7
commit 1bd59a0
Show file tree

Hide file tree

Showing 27 changed files with 2,648 additions and 32 deletions.
diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml
@@ -81,7 +81,6 @@ runs:
       with:
         agents: 3
         ingress_port: ${{ inputs.ingress_port }}
-        options: "--image docker.io/rancher/k3s:latest"
     - name: Check Kubernetes cluster
       shell: bash
       run: |

diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml
@@ -21,6 +21,10 @@ inputs:
     description: "k3d version"
     required: false
     default: "latest"
+  k3s_version:
+    description: "The k3s to use. The default version is `versions/K3S_VERSION`"
+    required: false
+    default: ""
   name:
     description: "Cluster name"
     required: false
@@ -52,6 +56,16 @@ runs:
       env:
         K3D_VERSION: ${{ inputs.version }}
 
+    - name: Detect k3s version
+      id: k3s_version
+      shell: bash
+      run: |
+        K3S_VERSION=${K3S_VERSION:-`cat versions/K3S_VERSION`}
+
+        echo "tag=${K3S_VERSION=$}" >> $GITHUB_OUTPUT
+      env:
+        K3S_VERSION: ${{ inputs.k3s_version }}
+
     - name: Install k3d
       shell: bash
       run: |
@@ -74,11 +88,13 @@ runs:
         if [ "${INGRESS_PORT}" != 0 ]; then
           OPTIONS="${OPTIONS} -p ${INGRESS_PORT}:80@loadbalancer"
         fi
+        OPTIONS="${OPTIONS} --image rancher/k3s:${K3S_VERSION}"
         echo "options=${OPTIONS}" >> $GITHUB_OUTPUT
       env:
         AGENTS: ${{ inputs.agents }}
         INGRESS_PORT: ${{ inputs.ingress_port }}
         OPTIONS: ${{ inputs.options }}
+        K3S_VERSION: ${{ steps.k3s_version.outputs.tag }}
 
     - name: Create k8s cluster
       shell: bash

diff --git a/.github/helm/values/values-lb.yaml b/.github/helm/values/values-lb.yaml
@@ -69,3 +69,9 @@ manager:
       auto_index_duration_limit: 2m
       auto_index_check_duration: 30s
       auto_index_length: 1000
+    corrector:
+      enabled: true
+      # suspend because you do not want corrector to start automatically in CI
+      # instead run it manually
+      suspend: true
+      schedule: "1 2 3 4 5"
diff --git a/.github/valdrelease/valdrelease.yaml b/.github/valdrelease/valdrelease.yaml
@@ -76,3 +76,7 @@ spec:
         auto_index_duration_limit: 2m
         auto_index_check_duration: 30s
         auto_index_length: 1000
+      corrector:
+        enabled: true
+        suspend: true
+        schedule: "1 2 3 4 5"
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -227,6 +227,42 @@ jobs:
         env:
           POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
 
+  e2e-jobs:
+    name: "E2E test (Jobs)"
+    needs: [dump-contexts-to-log]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set Git config
+        run: |
+          git config --global --add safe.directory ${GITHUB_WORKSPACE}
+
+      - name: Setup E2E environment
+        id: setup_e2e
+        uses: ./.github/actions/setup-e2e
+
+      - name: Deploy Vald
+        id: deploy_vald
+        uses: ./.github/actions/e2e-deploy-vald
+        with:
+          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
+          values: .github/helm/values/values-lb.yaml
+          wait_for_selector: app=vald-lb-gateway
+
+      - name: Run E2E Jobs
+        run: |
+          make hack/benchmark/assets/dataset/${{ env.DATASET }}
+          make E2E_BIND_PORT=8081 \
+            E2E_INSERT_COUNT=10000\
+            E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
+            E2E_TARGET_POD_NAME=${POD_NAME} \
+            E2E_TARGET_NAMESPACE=default \
+            e2e/index/job/correction
+        env:
+          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
+
   e2e-agent-and-sidecar:
     name: "E2E Agent & Sidecar test"
     needs: [dump-contexts-to-log]
@@ -278,6 +314,7 @@ jobs:
       - e2e-stream-crud
       - e2e-stream-crud-for-operator
       - e2e-stream-crud-skip-exist-check
+      - e2e-jobs
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

diff --git a/Makefile.d/build.mk b/Makefile.d/build.mk
@@ -209,7 +209,7 @@ cmd/manager/index/index: \
 cmd/index/job/correction/index-correction: \
 	$(GO_SOURCES_INTERNAL) \
 	$(PBGOS) \
-	$(shell find $(ROOTDIR)/cmd/index/job/correction/correction -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
+	$(shell find $(ROOTDIR)/cmd/index/job/correction -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go') \
 	$(shell find $(ROOTDIR)/pkg/index/job/correction -type f -name '*.go' -not -name '*_test.go' -not -name 'doc.go')
 	$(eval CGO_ENABLED = 0)
 	CGO_ENABLED=$(CGO_ENABLED) \

diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk
@@ -24,9 +24,10 @@ update/libs: \
 	update/helm-docs \
 	update/helm-operator \
 	update/jaeger-operator \
+	update/k3s \
 	update/kind \
-	update/kubectl \
 	update/kube-linter \
+	update/kubectl \
 	update/ngt \
 	update/prometheus-stack \
 	update/protobuf \
@@ -77,6 +78,11 @@ go/example/deps:
 update/chaos-mesh:
 	curl --silent https://api.github.com/repos/chaos-mesh/chaos-mesh/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' > $(ROOTDIR)/versions/CHAOS_MESH_VERSION
 
+.PHONY: update/k3s
+## update k3s version
+update/k3s:
+	curl --silent https://hub.docker.com/v2/repositories/rancher/k3s/tags | jq -r '.results[].name' | grep -E '.*-k3s1$$' | sort -V | tail -n 1 > $(ROOTDIR)/versions/K3S_VERSION
+
 .PHONY: update/go
 ## update go version
 update/go:

diff --git a/Makefile.d/e2e.mk b/Makefile.d/e2e.mk
@@ -69,6 +69,11 @@ e2e/remove/timestamp:
 e2e/insert/search:
 	$(call run-e2e-crud-test,-run TestE2EInsertAndSearch)
 
+.PHONY: e2e/index/job/correction
+## run index correction job e2e
+e2e/index/job/correction:
+	$(call run-e2e-crud-test,-run TestE2EIndexJobCorrection)
+
 .PHONY: e2e/maxdim
 ## run e2e/maxdim
 e2e/maxdim:
@@ -79,4 +84,3 @@ e2e/maxdim:
 e2e/sidecar:
 	$(call run-e2e-sidecar-test,-run TestE2EForSidecar)
 
-
diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk
@@ -23,22 +23,26 @@ k8s/manifest/clean:
 		k8s/agent \
 		k8s/discoverer \
 		k8s/gateway \
-		k8s/manager
+		k8s/manager \
+		k8s/index
 
 .PHONY: k8s/manifest/update
 ## update k8s manifests using helm templates
 k8s/manifest/update: \
 	k8s/manifest/clean
 	helm template \
-	    --values $(HELM_VALUES) \
-	    --output-dir $(TEMP_DIR) \
-	    charts/vald
+		--values $(HELM_VALUES) \
+		--set defaults.image.tag=$(VERSION) \
+		--output-dir $(TEMP_DIR) \
+		charts/vald
 	mkdir -p k8s/gateway
 	mkdir -p k8s/manager
+	mkdir -p k8s/index/job
 	mv $(TEMP_DIR)/vald/templates/agent k8s/agent
 	mv $(TEMP_DIR)/vald/templates/discoverer k8s/discoverer
 	mv $(TEMP_DIR)/vald/templates/gateway/lb k8s/gateway/lb
 	mv $(TEMP_DIR)/vald/templates/manager/index k8s/manager/index
+	mv $(TEMP_DIR)/vald/templates/index/job/correction k8s/index/job/correction
 	rm -rf $(TEMP_DIR)
 
 .PHONY: k8s/manifest/helm-operator/clean
@@ -80,6 +84,7 @@ k8s/vald/deploy:
 	kubectl apply -f $(TEMP_DIR)/vald/templates/agent || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/discoverer || true
 	kubectl apply -f $(TEMP_DIR)/vald/templates/gateway/lb || true
+	kubectl apply -f $(TEMP_DIR)/vald/templates/index/job/correction || true
 	rm -rf $(TEMP_DIR)
 	kubectl get pods -o jsonpath="{.items[*].spec.containers[*].image}" | tr " " "\n"
 
@@ -97,6 +102,7 @@ k8s/vald/delete:
 	    --set manager.index.image.repository=$(CRORG)/$(MANAGER_INDEX_IMAGE) \
 	    --output-dir $(TEMP_DIR) \
 	    charts/vald
+	kubectl delete -f $(TEMP_DIR)/vald/templates/index/job/correction
 	kubectl delete -f $(TEMP_DIR)/vald/templates/gateway/lb
 	kubectl delete -f $(TEMP_DIR)/vald/templates/manager/index
 	kubectl delete -f $(TEMP_DIR)/vald/templates/discoverer