Update dependencies, C++ standard, and improve Dockerfiles for better…

… build systems and localization (#2549) (#2557) Signed-off-by: kpango <[email protected]> Co-authored-by: Yusuke Kato <[email protected]>
vdaas · Aug 1, 2024 · 5023899 · 5023899
1 parent 063fbc6
commit 5023899
Show file tree

Hide file tree

Showing 62 changed files with 1,401 additions and 1,324 deletions.
diff --git a/.gitfiles b/.gitfiles
@@ -71,6 +71,7 @@
 .github/workflows/build-protobuf.yml
 .github/workflows/chatops-help.yml
 .github/workflows/chatops.yml
+.github/workflows/check-conflict.yml
 .github/workflows/codeql-analysis.yml
 .github/workflows/coverage.yml
 .github/workflows/detect-internal-config-changes.yml
@@ -117,6 +118,7 @@
 .github/workflows/semver-patch.yaml
 .github/workflows/test-hack.yml
 .github/workflows/unit-test.yaml
+.github/workflows/update-actions.yaml
 .github/workflows/update-protobuf.yaml
 .github/workflows/update-pull-request-and-issue-template.yml
 .github/workflows/update-web-docs.yml
@@ -544,7 +546,6 @@ dockers/manager/index/Dockerfile
 dockers/manager/index/README.md
 dockers/operator/helm/Dockerfile
 dockers/operator/helm/README.md
-dockers/readreplica/job/readreplica/rotate/Dockerfile
 dockers/template.md
 dockers/tools/benchmark/job/Dockerfile
 dockers/tools/benchmark/operator/Dockerfile
@@ -1971,6 +1972,7 @@ rust/libs/proto/src/sidecar.v1.rs
 rust/libs/proto/src/sidecar.v1.tonic.rs
 rust/libs/proto/src/vald.v1.rs
 rust/libs/proto/src/vald.v1.tonic.rs
+rust/rust-toolchain
 rust/rust-toolchain.toml
 tests/chaos/chart/.helmignore
 tests/chaos/chart/Chart.yaml
@@ -2000,6 +2002,7 @@ tests/e2e/sidecar/sidecar_test.go
 tests/performance/max_vector_dim_test.go
 versions/BUF_VERSION
 versions/CHAOS_MESH_VERSION
+versions/CMAKE_VERSION
 versions/DOCKER_VERSION
 versions/FAISS_VERSION
 versions/GOLANGCILINT_VERSION
@@ -2047,6 +2050,7 @@ versions/actions/GITHUB_CODEQL_ACTION_UPLOAD_SARIF
 versions/actions/GITHUB_ISSUE_METRICS
 versions/actions/MACHINE_LEARNING_APPS_ACTIONS_CHATOPS
 versions/actions/PETER_EVANS_CREATE_ISSUE_FROM_FILE
+versions/actions/PETER_EVANS_CREATE_PULL_REQUEST
 versions/actions/REVIEWDOG_ACTION_HADOLINT
 versions/actions/REVIEWDOG_ACTION_LANGUAGETOOL
 versions/actions/SHOGO82148_ACTIONS_UPLOAD_RELEASE_ASSET

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -24,7 +24,7 @@ assignees: ""
 
 - Vald Version: v1.7.12
 - Go Version: v1.22.5
-- Rust Version: v1.77.2
+- Rust Version: v1.80.0
 - Docker Version: v27.1.1
 - Kubernetes Version: v1.30.3
 - Helm Version: v3.15.3

diff --git a/.github/ISSUE_TEMPLATE/security_issue_report.md b/.github/ISSUE_TEMPLATE/security_issue_report.md
@@ -18,7 +18,7 @@ assignees: ""
 
 - Vald Version: v1.7.12
 - Go Version: v1.22.5
-- Rust Version: v1.77.2
+- Rust Version: v1.80.0
 - Docker Version: v27.1.1
 - Kubernetes Version: v1.30.3
 - Helm Version: v3.15.3

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -17,7 +17,7 @@
 <!--- Please change the versions below along with your environment -->
 - Vald Version: v1.7.12
 - Go Version: v1.22.5
-- Rust Version: v1.77.2
+- Rust Version: v1.80.0
 - Docker Version: v27.1.1
 - Kubernetes Version: v1.30.3
 - Helm Version: v3.15.3

diff --git a/.github/actions/setup-e2e/action.yaml b/.github/actions/setup-e2e/action.yaml
@@ -19,7 +19,7 @@ inputs:
   require_libhdf5:
     description: "If libhdf5 is not required, set this to false"
     required: false
-    default: "true"
+    default: "false"
   require_go:
     description: "If go is not required, set this to false"
     required: false
@@ -58,8 +58,8 @@ runs:
       if: ${{ inputs.require_libhdf5 == 'true' }}
       shell: bash
       run: |
-        sudo apt-get update
-        sudo apt-get install -y libhdf5-dev
+        apt-get update
+        apt-get install -y libhdf5-dev
     - name: Setup Go environment
       if: ${{ inputs.require_go == 'true' }}
       uses: ./.github/actions/setup-go

diff --git a/.github/actions/setup-k3d/action.yaml b/.github/actions/setup-k3d/action.yaml
@@ -57,9 +57,14 @@ runs:
       id: k3s_version
       shell: bash
       run: |
-        K3S_VERSION=${K3S_VERSION:-`cat versions/K3S_VERSION`}
-
-        echo "tag=${K3S_VERSION=$}" >> $GITHUB_OUTPUT
+        if [ -z "${K3S_VERSION}" ]; then
+          if [ -f "versions/K3S_VERSION" ]; then
+            K3S_VERSION=$(cat versions/K3S_VERSION)
+          else
+            K3S_VERSION=latest
+          fi
+        fi
+        echo "tag=${K3S_VERSION}" >> $GITHUB_OUTPUT
       env:
         K3S_VERSION: ${{ inputs.k3s_version }}
     - name: Install k3d
@@ -72,30 +77,38 @@ runs:
       shell: bash
       run: |
         k3d version
-    - name: Setup k3d options
+    - name: Create k8s cluster
       shell: bash
-      id: k3d_options
+      id: start_k3d
       run: |
-        if [ "${AGENTS}" != 0 ]; then
-          OPTIONS="${OPTIONS} --agents ${AGENTS}"
+        if [ "${K3D_INGRESS_PORT}" == "0" ] && [ -z "${K3D_OPTIONS}" ]; then
+          K3D_OPTIONS="--k3s-arg \"--disable=traefik@server:*\""
         fi
-        if [ "${INGRESS_PORT}" != 0 ]; then
-          OPTIONS="${OPTIONS} -p ${INGRESS_PORT}:80@loadbalancer"
+        if [ "${K3D_INGRESS_PORT}" != "0" ] && [ -n "${K3D_OPTIONS}" ]; then
+          K3D_OPTIONS="--port ${K3D_INGRESS_PORT}:80@loadbalancer ${K3D_OPTIONS}"
         fi
-        OPTIONS="${OPTIONS} --image rancher/k3s:${K3S_VERSION}"
-        echo "options=${OPTIONS}" >> $GITHUB_OUTPUT
+        make K3D_CLUSTER_NAME="${{ inputs.name }}" \
+            K3D_NODES="${{ inputs.agents }}" \
+            K3D_INGRESS_PORT="${K3D_INGRESS_PORT}" \
+            K3S_VERSION="${{ steps.k3s_version.outputs.tag }}" \
+            K3D_HOST="host.docker.internal" \
+            K3D_OPTIONS="${K3D_OPTIONS}" \
+            k3d/start
+        k3d cluster list
       env:
-        AGENTS: ${{ inputs.agents }}
-        INGRESS_PORT: ${{ inputs.ingress_port }}
-        OPTIONS: ${{ inputs.options }}
-        K3S_VERSION: ${{ steps.k3s_version.outputs.tag }}
-    - name: Create k8s cluster
+        K3D_INGRESS_PORT: ${{ inputs.ingress_port }}
+        K3D_OPTIONS: ${{ inputs.options }}
+    - name: Set k3d cluster config to KUBECONFIG
       shell: bash
-      id: start_k3d
       run: |
-        k3d cluster create ${{ inputs.name }} ${{ steps.k3d_options.outputs.options }}
-    - name: Ready k8s cluster
+        echo "KUBECONFIG=$(k3d kubeconfig write ${{ inputs.name }})" >> $GITHUB_ENV
+    - name: Check KUBECONFIG environment variable
+      shell: bash
+      run: |
+        echo $KUBECONFIG
+        cat $KUBECONFIG
+        cat /etc/hosts
+    - name: Show Kubernetes Cluster Info
       shell: bash
       run: |
-        k3d cluster list
         kubectl cluster-info dump
diff --git a/.github/helm/values/values-chaos.yaml b/.github/helm/values/values-chaos.yaml
@@ -35,7 +35,7 @@ gateway:
     enabled: true
     ingress:
       enabled: true
-      host: "localhost"
+      host: "host.docker.internal"
     service:
       # NOTE: https://doc.traefik.io/traefik/routing/providers/kubernetes-ingress/#on-service
       annotations:

diff --git a/.github/workflows/e2e-chaos.yaml b/.github/workflows/e2e-chaos.yaml
@@ -30,16 +30,21 @@ env:
   INGRESS_PORT: 8081
 jobs:
   dump-contexts-to-log:
-    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-chaos'
+    uses: ./.github/workflows/_detect-ci-container.yml
   agent-failure:
     name: "E2E chaos test (Agent failure: to test insert/search works even if one of the agents is failing)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -64,7 +69,7 @@ jobs:
       - name: Run Insert and Search operations
         run: |
           make hack/benchmark/assets/dataset/${{ env.DATASET }}
-          make E2E_BIND_HOST=localhost \
+          make E2E_BIND_HOST=host.docker.internal \
             E2E_BIND_PORT=8081 \
             E2E_PORTFORWARD_ENABLED=false \
             E2E_DATASET_NAME=${{ env.DATASET }} \
@@ -75,9 +80,12 @@ jobs:
             e2e/insert/search
   random-pod-failure:
     name: "E2E chaos test (random Pod failure: to test redundancy)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -102,7 +110,7 @@ jobs:
       - name: Run Insert and Search operations
         run: |
           make hack/benchmark/assets/dataset/${{ env.DATASET }}
-          make E2E_BIND_HOST=localhost \
+          make E2E_BIND_HOST=host.docker.internal \
             E2E_BIND_PORT=8081 \
             E2E_PORTFORWARD_ENABLED=false \
             E2E_DATASET_NAME=${{ env.DATASET }} \
@@ -113,9 +121,12 @@ jobs:
             e2e/insert/search
   agent-network-partition:
     name: "E2E chaos test (agent network partition: to test retries)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -140,7 +151,7 @@ jobs:
       - name: Run Insert and Search operations
         run: |
           make hack/benchmark/assets/dataset/${{ env.DATASET }}
-          make E2E_BIND_HOST=localhost \
+          make E2E_BIND_HOST=host.docker.internal \
             E2E_BIND_PORT=8081 \
             E2E_PORTFORWARD_ENABLED=false \
             E2E_DATASET_NAME=${{ env.DATASET }} \
@@ -151,9 +162,12 @@ jobs:
             e2e/insert/search
   clusterwide-network-bandwidth:
     name: "E2E chaos test (network bandwidth: to test it works properly under bandwidth limitation)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -178,7 +192,7 @@ jobs:
       - name: Run Insert and Search operations
         run: |
           make hack/benchmark/assets/dataset/${{ env.DATASET }}
-          make E2E_BIND_HOST=localhost \
+          make E2E_BIND_HOST=host.docker.internal \
             E2E_BIND_PORT=8081 \
             E2E_PORTFORWARD_ENABLED=false \
             E2E_DATASET_NAME=${{ env.DATASET }} \

diff --git a/.github/workflows/e2e-code-bench-agent.yaml b/.github/workflows/e2e-code-bench-agent.yaml
@@ -58,6 +58,7 @@ jobs:
     needs: [detect-ci-container]
     container:
       image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -85,6 +86,7 @@ jobs:
     needs: [detect-ci-container]
     container:
       image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/e2e-max-dim.yml b/.github/workflows/e2e-max-dim.yml
@@ -27,16 +27,21 @@ on:
       - "labeled"
 jobs:
   dump-contexts-to-log:
-    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-max-dim'
+    uses: ./.github/workflows/_detect-ci-container.yml
   e2e-max-dimension-insert:
     name: "E2E test (Max Dimension Insert: skip strict exist check)"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config

diff --git a/.github/workflows/e2e-profiling.yml b/.github/workflows/e2e-profiling.yml
@@ -28,16 +28,21 @@ env:
   DATASET: fashion-mnist-784-euclidean.hdf5
 jobs:
   dump-contexts-to-log:
-    if: startsWith( github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
+  detect-ci-container:
+    if: startsWith(github.ref, 'refs/tags/') || github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling'
+    uses: ./.github/workflows/_detect-ci-container.yml
   e2e-profiling:
     name: "E2E profiling"
-    needs: [dump-contexts-to-log]
+    needs: [detect-ci-container]
     runs-on: ubuntu-latest
     timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
     steps:
       - uses: actions/checkout@v4
       - name: Set Git config
@@ -100,13 +105,27 @@ jobs:
           path: ./profiles-main
           key: ${{ runner.os }}-profiles-main-${{ github.sha }}
           restore-keys: ${{ runner.os }}-profiles-main-
-      - name: Install dependencies
+      - name: Update cache
+        if: startsWith(github.ref, 'refs/tags/')
         run: |
-          sudo apt update
-          sudo apt install -y graphviz
+          mkdir -p profiles-main
+          cp -f profiles/* profiles-main/
+          cp -f versions/VALD_VERSION profiles-main/
       - name: Generate graphs
+        shell: bash
         run: |
-          tag=$(cat profiles-main/VALD_VERSION || echo "unknown")
+          tag="unknown"
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            tag="pr-${{ github.event.pull_request.number }}"
+          elif [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            tag="main-${{ github.sha }}"
+          elif [[ "${{ github.ref }}" == "refs/heads/release/*" ]]; then
+            version=$(echo ${{ github.ref }} | sed 's/refs\/heads\/release\///')
+            tag="v${version}-${{ github.sha }}"
+          elif [[ -f profiles-main/VALD_VERSION && -s profiles-main/VALD_VERSION ]]; then
+            tag=$(cat profiles-main/VALD_VERSION)
+          fi
+
           mkdir -p graphs
           for svc in vald-agent-ngt vald-lb-gateway vald-discoverer vald-manager-index
           do
@@ -130,12 +149,6 @@ jobs:
         with:
           name: graphs
           path: graphs/*
-      - name: Update cache
-        if: startsWith( github.ref, 'refs/tags/')
-        run: |
-          mkdir -p profiles-main
-          cp -f profiles/* profiles-main/
-          cp -f versions/VALD_VERSION profiles-main/
       - name: Upload to vald-ci-images repository
         if: github.event.action == 'labeled' && github.event.label.name == 'actions/e2e-profiling'
         run: |