From 6573794781697d49b38f3537f528e1e595e5d3ca Mon Sep 17 00:00:00 2001 From: ykadowak Date: Fri, 19 Jan 2024 02:24:52 +0000 Subject: [PATCH] Add support for Minikube environment setup and add readreplica e2e --- .../e2e-deploy-vald-readreplica/action.yaml | 128 ++++++++++++++++++ .github/actions/setup-e2e/action.yaml | 9 ++ .github/helm/values/values-readreplica.yaml | 82 +++++++++++ .github/workflows/e2e.yml | 48 +++++++ 4 files changed, 267 insertions(+) create mode 100644 .github/actions/e2e-deploy-vald-readreplica/action.yaml create mode 100644 .github/helm/values/values-readreplica.yaml diff --git a/.github/actions/e2e-deploy-vald-readreplica/action.yaml b/.github/actions/e2e-deploy-vald-readreplica/action.yaml new file mode 100644 index 00000000000..4bbe1316c52 --- /dev/null +++ b/.github/actions/e2e-deploy-vald-readreplica/action.yaml @@ -0,0 +1,128 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +name: "Deploy Vald Read Replica for E2E test" +description: "A action to deploy vald read replica for E2E test" + +inputs: + require_minio: + description: "If Minio is required, set this to true." + required: false + default: "false" + helm_extra_options: + description: "Extra options that passed to Helm command." + required: false + default: "" + values: + description: "Path to the values.yaml that passed to Helm command." + required: false + default: "false" + wait_for_selector: + description: "Label selector used for specifying a pod waited for" + required: false + default: "app=vald-lb-gateway" + wait_for_timeout: + description: "Timeout used for waiting for pods" + required: false + default: "600s" + use_local_charts: + description: "If you want to use local charts, set this to true." + required: false + default: "true" + default_image_tag: + description: "Default image tag. e.g) nightly, vx.x, vx.x.x" + required: true + default: "nightly" +outputs: + POD_NAME: + description: "A pod name that waited for" + value: ${{ steps.get_real_pod_name.outputs.POD_NAME }} + +runs: + using: "composite" + steps: + - name: Deploy Minio + id: deploy_minio + shell: bash + if: ${{ inputs.require_minio == 'true' }} + run: | + make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy + + - name: Dump Helm values + shell: bash + run: | + cat ${{ inputs.values }} + + - name: Deploy vald read replica from remote charts + shell: bash + id: deploy_vald_readreplica_remote + if: ${{ inputs.use_local_charts == 'false' }} + run: | + helm install \ + --values ${VALUES} \ + --set defaults.image.tag=${DEFAULT_IMAGE_TAG} \ + ${HELM_EXTRA_OPTIONS} \ + --generate-name charts/vald-readreplica + + sleep 3 + + kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} + + kubectl get pods + + podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'` + echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT + env: + DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }} + VALUES: ${{ inputs.values }} + HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} + WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }} + WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }} + + - name: Deploy vald read replica from local charts + shell: bash + id: deploy_vald_readreplica_local + if: ${{ inputs.use_local_charts == 'true' }} + run: | + make k8s/vald-readreplica/deploy VERSION=${DEFAULT_IMAGE_TAG} HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}" + + sleep 3 + + kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT} + + kubectl get pods + + podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'` + echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT + env: + DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }} + VALUES: ${{ inputs.values }} + HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }} + WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }} + WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }} + + - name: Get real pod name + shell: bash + id: get_real_pod_name + env: + PODNAME_LOCAL_DEPLOY: ${{ steps.deploy_vald_readreplica_local.outputs.POD_NAME }} + PODNAME_REMOTE_DEPLOY: ${{ steps.deploy_vald_readreplica_remote.outputs.POD_NAME }} + # Set GITHUB_OUTPUT to the not empty one, PODNAME_LOCAL_DEPLOY or PODNAME_REMOTE_DEPLOY + run: | + if [[ -n "${PODNAME_LOCAL_DEPLOY}" ]]; then + echo "POD_NAME=${PODNAME_LOCAL_DEPLOY}" >> $GITHUB_OUTPUT + else + echo "POD_NAME=${PODNAME_REMOTE_DEPLOY}" >> $GITHUB_OUTPUT + fi diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml index a2a17bbcef1..f24319d6fe4 100644 --- a/.github/actions/setup-e2e/action.yaml +++ b/.github/actions/setup-e2e/action.yaml @@ -33,6 +33,10 @@ inputs: description: "If k3d is not required, set this to false" required: false default: "true" + require_minikube: + description: "If minikube is not required, set this to true and set require_k3d to false" + required: false + default: "false" ingress_port: description: 'If it is not "0", ingress will be exposed to the specified port' required: false @@ -94,6 +98,11 @@ runs: agents: 3 ingress_port: ${{ inputs.ingress_port }} + # TODO: minikubeでもagentsやingress_portのようなオプションは必要なのか + - name: Setup Minikube environment + if: ${{ inputs.require_minikube == 'true' }} + uses: medyagh/setup-minikube@master + - name: Check Kubernetes cluster shell: bash run: | diff --git a/.github/helm/values/values-readreplica.yaml b/.github/helm/values/values-readreplica.yaml new file mode 100644 index 00000000000..2b85368cffd --- /dev/null +++ b/.github/helm/values/values-readreplica.yaml @@ -0,0 +1,82 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defaults: + logging: + level: info + networkPolicy: + enabled: true + +gateway: + lb: + enabled: true + minReplicas: 1 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + gateway_config: + index_replica: 3 + +agent: + minReplicas: 3 + maxReplicas: 10 + podManagementPolicy: Parallel + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + ngt: + auto_index_duration_limit: 2m + auto_index_check_duration: 30s + auto_index_length: 1000 + dimension: 784 + persistentVolume: + enabled: true + # For local-path-provisioner, we cannot use ReadWriteOncePod because it is not supported. + accessMode: ReadWriteOnce + storageClass: csi-hostpath-sc + size: 1Gi + readreplica: + enabled: true + snapshot_classname: "csi-hostpath-snapclass" + replica: 1 + + +discoverer: + minReplicas: 1 + hpa: + enabled: false + resources: + requests: + cpu: 100m + memory: 50Mi + +manager: + index: + replicas: 1 + resources: + requests: + cpu: 100m + memory: 30Mi + indexer: + auto_index_duration_limit: 2m + auto_index_check_duration: 30s + auto_index_length: 1000 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 28f0722a11d..49aa08880ea 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -319,6 +319,54 @@ jobs: env: POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + # TODO: readreplicaの再起動ロジック等をどこかに入れる + e2e-stream-crud-with-readreplica: + name: "E2E test (Stream CRUD with read replica)" + needs: [dump-contexts-to-log] + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Set Git config + run: | + git config --global --add safe.directory ${GITHUB_WORKSPACE} + + - name: Setup E2E environment + id: setup_e2e + uses: ./.github/actions/setup-e2e + with: + require_k3d: "false" + require_minikube: "true" + + - name: Deploy Vald + id: deploy_vald + uses: ./.github/actions/e2e-deploy-vald-readreplica + with: + default_image_tag: ${{ steps.setup_e2e.outputs.DEFAULT_IMAGE_TAG }} + helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }} + values: .github/helm/values/values-readreplica.yaml + wait_for_selector: app=vald-lb-gateway + + - name: Run E2E CRUD + run: | + make hack/benchmark/assets/dataset/${{ env.DATASET }} + make E2E_BIND_PORT=8081 \ + E2E_DATASET_NAME=${{ env.DATASET }} \ + E2E_INSERT_COUNT=10000\ + E2E_SEARCH_COUNT=10000 \ + E2E_SEARCH_BY_ID_COUNT=10000 \ + E2E_GET_OBJECT_COUNT=100 \ + E2E_UPDATE_COUNT=100 \ + E2E_UPSERT_COUNT=100 \ + E2E_REMOVE_COUNT=100 \ + E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \ + E2E_TARGET_POD_NAME=${POD_NAME} \ + E2E_TARGET_NAMESPACE=default \ + e2e + env: + POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }} + e2e-agent-and-sidecar: name: "E2E Agent & Sidecar test" needs: [dump-contexts-to-log]