Skip to content

Commit

Permalink
Add support for Minikube environment setup and add readreplica e2e
Browse files Browse the repository at this point in the history
  • Loading branch information
ykadowak committed Jan 19, 2024
1 parent aa5d05d commit 6573794
Show file tree
Hide file tree
Showing 4 changed files with 267 additions and 0 deletions.
128 changes: 128 additions & 0 deletions .github/actions/e2e-deploy-vald-readreplica/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
#
# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: "Deploy Vald Read Replica for E2E test"
description: "A action to deploy vald read replica for E2E test"

inputs:
require_minio:
description: "If Minio is required, set this to true."
required: false
default: "false"
helm_extra_options:
description: "Extra options that passed to Helm command."
required: false
default: ""
values:
description: "Path to the values.yaml that passed to Helm command."
required: false
default: "false"
wait_for_selector:
description: "Label selector used for specifying a pod waited for"
required: false
default: "app=vald-lb-gateway"
wait_for_timeout:
description: "Timeout used for waiting for pods"
required: false
default: "600s"
use_local_charts:
description: "If you want to use local charts, set this to true."
required: false
default: "true"
default_image_tag:
description: "Default image tag. e.g) nightly, vx.x, vx.x.x"
required: true
default: "nightly"
outputs:
POD_NAME:
description: "A pod name that waited for"
value: ${{ steps.get_real_pod_name.outputs.POD_NAME }}

runs:
using: "composite"
steps:
- name: Deploy Minio
id: deploy_minio
shell: bash
if: ${{ inputs.require_minio == 'true' }}
run: |
make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy
- name: Dump Helm values
shell: bash
run: |
cat ${{ inputs.values }}
- name: Deploy vald read replica from remote charts
shell: bash
id: deploy_vald_readreplica_remote
if: ${{ inputs.use_local_charts == 'false' }}
run: |
helm install \
--values ${VALUES} \
--set defaults.image.tag=${DEFAULT_IMAGE_TAG} \
${HELM_EXTRA_OPTIONS} \
--generate-name charts/vald-readreplica
sleep 3
kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT}
kubectl get pods
podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
env:
DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
VALUES: ${{ inputs.values }}
HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}
WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }}

- name: Deploy vald read replica from local charts
shell: bash
id: deploy_vald_readreplica_local
if: ${{ inputs.use_local_charts == 'true' }}
run: |
make k8s/vald-readreplica/deploy VERSION=${DEFAULT_IMAGE_TAG} HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
sleep 3
kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT}
kubectl get pods
podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
env:
DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
VALUES: ${{ inputs.values }}
HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}
WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }}

- name: Get real pod name
shell: bash
id: get_real_pod_name
env:
PODNAME_LOCAL_DEPLOY: ${{ steps.deploy_vald_readreplica_local.outputs.POD_NAME }}
PODNAME_REMOTE_DEPLOY: ${{ steps.deploy_vald_readreplica_remote.outputs.POD_NAME }}
# Set GITHUB_OUTPUT to the not empty one, PODNAME_LOCAL_DEPLOY or PODNAME_REMOTE_DEPLOY
run: |
if [[ -n "${PODNAME_LOCAL_DEPLOY}" ]]; then
echo "POD_NAME=${PODNAME_LOCAL_DEPLOY}" >> $GITHUB_OUTPUT
else
echo "POD_NAME=${PODNAME_REMOTE_DEPLOY}" >> $GITHUB_OUTPUT
fi
9 changes: 9 additions & 0 deletions .github/actions/setup-e2e/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ inputs:
description: "If k3d is not required, set this to false"
required: false
default: "true"
require_minikube:
description: "If minikube is not required, set this to true and set require_k3d to false"
required: false
default: "false"
ingress_port:
description: 'If it is not "0", ingress will be exposed to the specified port'
required: false
Expand Down Expand Up @@ -94,6 +98,11 @@ runs:
agents: 3
ingress_port: ${{ inputs.ingress_port }}

# TODO: minikubeでもagentsやingress_portのようなオプションは必要なのか
- name: Setup Minikube environment
if: ${{ inputs.require_minikube == 'true' }}
uses: medyagh/setup-minikube@master

- name: Check Kubernetes cluster
shell: bash
run: |
Expand Down
82 changes: 82 additions & 0 deletions .github/helm/values/values-readreplica.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
#
# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

defaults:
logging:
level: info
networkPolicy:
enabled: true

gateway:
lb:
enabled: true
minReplicas: 1
hpa:
enabled: false
resources:
requests:
cpu: 100m
memory: 50Mi
gateway_config:
index_replica: 3

agent:
minReplicas: 3
maxReplicas: 10
podManagementPolicy: Parallel
hpa:
enabled: false
resources:
requests:
cpu: 100m
memory: 50Mi
ngt:
auto_index_duration_limit: 2m
auto_index_check_duration: 30s
auto_index_length: 1000
dimension: 784
persistentVolume:
enabled: true
# For local-path-provisioner, we cannot use ReadWriteOncePod because it is not supported.
accessMode: ReadWriteOnce
storageClass: csi-hostpath-sc
size: 1Gi
readreplica:
enabled: true
snapshot_classname: "csi-hostpath-snapclass"
replica: 1


discoverer:
minReplicas: 1
hpa:
enabled: false
resources:
requests:
cpu: 100m
memory: 50Mi

manager:
index:
replicas: 1
resources:
requests:
cpu: 100m
memory: 30Mi
indexer:
auto_index_duration_limit: 2m
auto_index_check_duration: 30s
auto_index_length: 1000
48 changes: 48 additions & 0 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,54 @@ jobs:
env:
POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}

# TODO: readreplicaの再起動ロジック等をどこかに入れる
e2e-stream-crud-with-readreplica:
name: "E2E test (Stream CRUD with read replica)"
needs: [dump-contexts-to-log]
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v4

- name: Set Git config
run: |
git config --global --add safe.directory ${GITHUB_WORKSPACE}
- name: Setup E2E environment
id: setup_e2e
uses: ./.github/actions/setup-e2e
with:
require_k3d: "false"
require_minikube: "true"

- name: Deploy Vald
id: deploy_vald
uses: ./.github/actions/e2e-deploy-vald-readreplica
with:
default_image_tag: ${{ steps.setup_e2e.outputs.DEFAULT_IMAGE_TAG }}
helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
values: .github/helm/values/values-readreplica.yaml
wait_for_selector: app=vald-lb-gateway

- name: Run E2E CRUD
run: |
make hack/benchmark/assets/dataset/${{ env.DATASET }}
make E2E_BIND_PORT=8081 \
E2E_DATASET_NAME=${{ env.DATASET }} \
E2E_INSERT_COUNT=10000\
E2E_SEARCH_COUNT=10000 \
E2E_SEARCH_BY_ID_COUNT=10000 \
E2E_GET_OBJECT_COUNT=100 \
E2E_UPDATE_COUNT=100 \
E2E_UPSERT_COUNT=100 \
E2E_REMOVE_COUNT=100 \
E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
E2E_TARGET_POD_NAME=${POD_NAME} \
E2E_TARGET_NAMESPACE=default \
e2e
env:
POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}

e2e-agent-and-sidecar:
name: "E2E Agent & Sidecar test"
needs: [dump-contexts-to-log]
Expand Down

0 comments on commit 6573794

Please sign in to comment.