Merge branch 'main' into feature/workflow/add-dispatch-wf-for-udpate-web

.github/actions/detect-docker-image-tags/action.yaml

@@ -48,6 +48,7 @@ runs:
           ["vdaas/vald-agent-sidecar"]="agent.sidecar.image.tag"
           ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag"
           ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag"
+          ["vdaas/vald-mirror-gateway"]="gateway.mirror.image.tag"
           ["vdaas/vald-manager-index"]="manager.index.image.tag"
           ["vdaas/vald-index-creation"]="manager.index.creator.image.tag"
           ["vdaas/vald-index-save"]="manager.index.saver.image.tag"

.github/actions/e2e-deploy-vald-helm-operator/action.yaml

@@ -75,7 +75,7 @@ runs:
           ${HELM_EXTRA_OPTIONS} \
           charts/vald-helm-operator/.
 
-        sleep 3
+        sleep 6
       env:
         DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
@@ -97,7 +97,7 @@ runs:
       run: |
         kubectl apply -f ${VALDRELEASE}
 
-        sleep 3
+        sleep 6
 
         kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT}
 

.github/actions/e2e-deploy-vald-readreplica/action.yaml

@@ -0,0 +1,128 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Deploy Vald Read Replica for E2E test"
+description: "A action to deploy vald read replica for E2E test"
+
+inputs:
+  require_minio:
+    description: "If Minio is required, set this to true."
+    required: false
+    default: "false"
+  helm_extra_options:
+    description: "Extra options that passed to Helm command."
+    required: false
+    default: ""
+  values:
+    description: "Path to the values.yaml that passed to Helm command."
+    required: false
+    default: "false"
+  wait_for_selector:
+    description: "Label selector used for specifying a pod waited for"
+    required: false
+    default: "app=vald-lb-gateway"
+  wait_for_timeout:
+    description: "Timeout used for waiting for pods"
+    required: false
+    default: "600s"
+  use_local_charts:
+    description: "If you want to use local charts, set this to true."
+    required: false
+    default: "true"
+  default_image_tag:
+    description: "Default image tag. e.g) nightly, vx.x, vx.x.x"
+    required: true
+    default: "nightly"
+outputs:
+  POD_NAME:
+    description: "A pod name that waited for"
+    value: ${{ steps.get_real_pod_name.outputs.POD_NAME }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Deploy Minio
+      id: deploy_minio
+      shell: bash
+      if: ${{ inputs.require_minio == 'true' }}
+      run: |
+        make K8S_SLEEP_DURATION_FOR_WAIT_COMMAND=10 k8s/external/minio/deploy
+
+    - name: Dump Helm values
+      shell: bash
+      run: |
+        cat ${{ inputs.values }}
+
+    - name: Deploy vald read replica from remote charts
+      shell: bash
+      id: deploy_vald_readreplica_remote
+      if: ${{ inputs.use_local_charts == 'false' }}
+      run: |
+        helm install \
+          --values ${VALUES} \
+          --set defaults.image.tag=${DEFAULT_IMAGE_TAG} \
+          ${HELM_EXTRA_OPTIONS} \
+          --generate-name charts/vald-readreplica
+
+        sleep 3
+
+        kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT}
+
+        kubectl get pods
+
+        podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
+        echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
+      env:
+        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
+        VALUES: ${{ inputs.values }}
+        HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
+        WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}
+        WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }}
+
+    - name: Deploy vald read replica from local charts
+      shell: bash
+      id: deploy_vald_readreplica_local
+      if: ${{ inputs.use_local_charts == 'true' }}
+      run: |
+        make k8s/vald-readreplica/deploy VERSION=${DEFAULT_IMAGE_TAG} HELM_VALUES=${VALUES} HELM_EXTRA_OPTIONS="${HELM_EXTRA_OPTIONS}"
+
+        sleep 3
+
+        kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT}
+
+        kubectl get pods
+
+        podname=`kubectl get pods --selector=${WAIT_FOR_SELECTOR} | tail -1 | awk '{print $1}'`
+        echo "POD_NAME=${podname}" >> $GITHUB_OUTPUT
+      env:
+        DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
+        VALUES: ${{ inputs.values }}
+        HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
+        WAIT_FOR_SELECTOR: ${{ inputs.wait_for_selector }}
+        WAIT_FOR_TIMEOUT: ${{ inputs.wait_for_timeout }}
+
+    - name: Get real pod name
+      shell: bash
+      id: get_real_pod_name
+      env:
+        PODNAME_LOCAL_DEPLOY: ${{ steps.deploy_vald_readreplica_local.outputs.POD_NAME }}
+        PODNAME_REMOTE_DEPLOY: ${{ steps.deploy_vald_readreplica_remote.outputs.POD_NAME }}
+      # Set GITHUB_OUTPUT to the not empty one, PODNAME_LOCAL_DEPLOY or PODNAME_REMOTE_DEPLOY
+      run: |
+        if [[ -n "${PODNAME_LOCAL_DEPLOY}" ]]; then
+          echo "POD_NAME=${PODNAME_LOCAL_DEPLOY}" >> $GITHUB_OUTPUT
+        else
+          echo "POD_NAME=${PODNAME_REMOTE_DEPLOY}" >> $GITHUB_OUTPUT
+        fi

.github/actions/e2e-deploy-vald/action.yaml

@@ -29,6 +29,9 @@ inputs:
     description: "Path to the values.yaml that passed to Helm command."
     required: false
     default: "false"
+  namespace:
+    description: "Namespace to deploy."
+    required: false
   wait_for_selector:
     description: "Label selector used for specifying a pod waited for"
     required: false
@@ -65,6 +68,14 @@ runs:
       run: |
         cat ${{ inputs.values }}
 
+    - name: Change namespace
+      if: ${{ inputs.namespace != '' }}
+      shell: bash
+      run: |
+        kubectl create ns ${NAMESPACE} ; kubectl config set-context --current --namespace=${NAMESPACE}
+      env:
+        NAMESPACE: ${{ inputs.namespace }}
+
     - name: Deploy vald from remote charts
       shell: bash
       id: deploy_vald_remote

.github/actions/setup-e2e/action.yaml

@@ -33,6 +33,10 @@ inputs:
     description: "If k3d is not required, set this to false"
     required: false
     default: "true"
+  require_minikube:
+    description: "If minikube is not required, set this to true and set require_k3d to false"
+    required: false
+    default: "false"
   ingress_port:
     description: 'If it is not "0", ingress will be exposed to the specified port'
     required: false
@@ -94,6 +98,13 @@ runs:
         agents: 3
         ingress_port: ${{ inputs.ingress_port }}
 
+    - name: Setup Minikube environment
+      if: ${{ inputs.require_minikube == 'true' }}
+      shell: bash
+      run: |
+        make minikube/install
+        make minikube/start
+
     - name: Check Kubernetes cluster
       shell: bash
       run: |

.github/helm/values/vald-mirror-target.yaml

@@ -0,0 +1,24 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: vald.vdaas.org/v1
+kind: ValdMirrorTarget
+metadata:
+  name: mirror-target-02
+spec:
+  colocation: dc1
+  target:
+    host: vald-mirror-gateway.vald-02.svc.cluster.local
+    port: 8081

.github/helm/values/values-mirror-01.yaml

@@ -0,0 +1,81 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+defaults:
+  logging:
+    level: info
+  networkPolicy:
+    enabled: false
+
+gateway:
+  mirror:
+    enabled: true
+    clusterRoleBinding:
+      name: vald-mirror-01
+    serviceAccount:
+      name: vald-mirror-01
+  lb:
+    enabled: true
+    minReplicas: 1
+    hpa:
+      enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    gateway_config:
+      index_replica: 3
+
+agent:
+  minReplicas: 3
+  maxReplicas: 10
+  podManagementPolicy: Parallel
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  ngt:
+    auto_index_duration_limit: 2m
+    auto_index_check_duration: 30s
+    auto_index_length: 1000
+    dimension: 784
+
+discoverer:
+  minReplicas: 1
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  clusterRoleBinding:
+    name: vald-01
+  serviceAccount:
+    name: vald-01
+
+manager:
+  index:
+    replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 30Mi
+    indexer:
+      auto_index_duration_limit: 2m
+      auto_index_check_duration: 30s
+      auto_index_length: 1000

.github/helm/values/values-mirror-02.yaml

@@ -0,0 +1,85 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+defaults:
+  logging:
+    level: info
+  networkPolicy:
+    enabled: false
+
+gateway:
+  mirror:
+    enabled: true
+    clusterRole:
+      enabled: false
+    clusterRoleBinding:
+      name: vald-mirror-02
+    serviceAccount:
+      name: vald-mirror-02
+  lb:
+    enabled: true
+    minReplicas: 1
+    hpa:
+      enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    gateway_config:
+      index_replica: 3
+
+agent:
+  minReplicas: 3
+  maxReplicas: 10
+  podManagementPolicy: Parallel
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  ngt:
+    auto_index_duration_limit: 2m
+    auto_index_check_duration: 30s
+    auto_index_length: 1000
+    dimension: 784
+
+discoverer:
+  minReplicas: 1
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  clusterRole:
+    enabled: false
+  clusterRoleBinding:
+    name: vald-02
+  serviceAccount:
+    name: vald-02
+
+manager:
+  index:
+    replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 30Mi
+    indexer:
+      auto_index_duration_limit: 2m
+      auto_index_check_duration: 30s
+      auto_index_length: 1000