Merge branch 'main' into feature/readreplica/client

vdaas · Jan 19, 2024 · 115d09a · 115d09a
2 parents aa5d05d + 306f0ef
commit 115d09a
Show file tree

Hide file tree

Showing 125 changed files with 22,166 additions and 671 deletions.
diff --git a/.github/actions/detect-docker-image-tags/action.yaml b/.github/actions/detect-docker-image-tags/action.yaml
@@ -48,6 +48,7 @@ runs:
           ["vdaas/vald-agent-sidecar"]="agent.sidecar.image.tag"
           ["vdaas/vald-discoverer-k8s"]="discoverer.image.tag"
           ["vdaas/vald-lb-gateway"]="gateway.lb.image.tag"
+          ["vdaas/vald-mirror-gateway"]="gateway.mirror.image.tag"
           ["vdaas/vald-manager-index"]="manager.index.image.tag"
           ["vdaas/vald-index-creation"]="manager.index.creator.image.tag"
           ["vdaas/vald-index-save"]="manager.index.saver.image.tag"

diff --git a/.github/actions/e2e-deploy-vald-helm-operator/action.yaml b/.github/actions/e2e-deploy-vald-helm-operator/action.yaml
@@ -75,7 +75,7 @@ runs:
           ${HELM_EXTRA_OPTIONS} \
           charts/vald-helm-operator/.
 
-        sleep 3
+        sleep 6
       env:
         DEFAULT_IMAGE_TAG: ${{ inputs.default_image_tag }}
         HELM_EXTRA_OPTIONS: ${{ inputs.helm_extra_options }}
@@ -97,7 +97,7 @@ runs:
       run: |
         kubectl apply -f ${VALDRELEASE}
 
-        sleep 3
+        sleep 6
 
         kubectl wait --for=condition=ready pod -l ${WAIT_FOR_SELECTOR} --timeout=${WAIT_FOR_TIMEOUT}
 

diff --git a/.github/actions/e2e-deploy-vald/action.yaml b/.github/actions/e2e-deploy-vald/action.yaml
@@ -29,6 +29,9 @@ inputs:
     description: "Path to the values.yaml that passed to Helm command."
     required: false
     default: "false"
+  namespace:
+    description: "Namespace to deploy."
+    required: false
   wait_for_selector:
     description: "Label selector used for specifying a pod waited for"
     required: false
@@ -65,6 +68,14 @@ runs:
       run: |
         cat ${{ inputs.values }}
 
+    - name: Change namespace
+      if: ${{ inputs.namespace != '' }}
+      shell: bash
+      run: |
+        kubectl create ns ${NAMESPACE} ; kubectl config set-context --current --namespace=${NAMESPACE}
+      env:
+        NAMESPACE: ${{ inputs.namespace }}
+
     - name: Deploy vald from remote charts
       shell: bash
       id: deploy_vald_remote

diff --git a/.github/helm/values/vald-mirror-target.yaml b/.github/helm/values/vald-mirror-target.yaml
@@ -0,0 +1,24 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: vald.vdaas.org/v1
+kind: ValdMirrorTarget
+metadata:
+  name: mirror-target-02
+spec:
+  colocation: dc1
+  target:
+    host: vald-mirror-gateway.vald-02.svc.cluster.local
+    port: 8081
diff --git a/.github/helm/values/values-mirror-01.yaml b/.github/helm/values/values-mirror-01.yaml
@@ -0,0 +1,81 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+defaults:
+  logging:
+    level: info
+  networkPolicy:
+    enabled: false
+
+gateway:
+  mirror:
+    enabled: true
+    clusterRoleBinding:
+      name: vald-mirror-01
+    serviceAccount:
+      name: vald-mirror-01
+  lb:
+    enabled: true
+    minReplicas: 1
+    hpa:
+      enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    gateway_config:
+      index_replica: 3
+
+agent:
+  minReplicas: 3
+  maxReplicas: 10
+  podManagementPolicy: Parallel
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  ngt:
+    auto_index_duration_limit: 2m
+    auto_index_check_duration: 30s
+    auto_index_length: 1000
+    dimension: 784
+
+discoverer:
+  minReplicas: 1
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  clusterRoleBinding:
+    name: vald-01
+  serviceAccount:
+    name: vald-01
+
+manager:
+  index:
+    replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 30Mi
+    indexer:
+      auto_index_duration_limit: 2m
+      auto_index_check_duration: 30s
+      auto_index_length: 1000
diff --git a/.github/helm/values/values-mirror-02.yaml b/.github/helm/values/values-mirror-02.yaml
@@ -0,0 +1,85 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+defaults:
+  logging:
+    level: info
+  networkPolicy:
+    enabled: false
+
+gateway:
+  mirror:
+    enabled: true
+    clusterRole:
+      enabled: false
+    clusterRoleBinding:
+      name: vald-mirror-02
+    serviceAccount:
+      name: vald-mirror-02
+  lb:
+    enabled: true
+    minReplicas: 1
+    hpa:
+      enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    gateway_config:
+      index_replica: 3
+
+agent:
+  minReplicas: 3
+  maxReplicas: 10
+  podManagementPolicy: Parallel
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  ngt:
+    auto_index_duration_limit: 2m
+    auto_index_check_duration: 30s
+    auto_index_length: 1000
+    dimension: 784
+
+discoverer:
+  minReplicas: 1
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  clusterRole:
+    enabled: false
+  clusterRoleBinding:
+    name: vald-02
+  serviceAccount:
+    name: vald-02
+
+manager:
+  index:
+    replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 30Mi
+    indexer:
+      auto_index_duration_limit: 2m
+      auto_index_check_duration: 30s
+      auto_index_length: 1000
diff --git a/.github/workflows/dockers-gateway-mirror-image.yaml b/.github/workflows/dockers-gateway-mirror-image.yaml
@@ -0,0 +1,82 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Build docker image: gateway-mirror"
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/dockers-gateway-mirror-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/gateway/mirror/**"
+      - "cmd/gateway/mirror/**"
+      - "pkg/gateway/internal/**"
+      - "dockers/gateway/mirror/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/dockers-gateway-mirror-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/gateway/mirror/**"
+      - "cmd/gateway/mirror/**"
+      - "pkg/gateway/internal/**"
+      - "dockers/gateway/mirror/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/dockers-gateway-mirror-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/**/*_mock.go"
+      - "!internal/db/**"
+      - "!internal/k8s/**"
+      - "apis/grpc/**"
+      - "pkg/gateway/mirror/**"
+      - "cmd/gateway/nirror/**"
+      - "pkg/gateway/internal/**"
+      - "dockers/gateway/mirror/Dockerfile"
+      - "versions/GO_VERSION"
+
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: gateway-mirror
+    secrets: inherit