Sync release/v1.7 to main

.gitfiles

@@ -75,6 +75,7 @@
 .github/workflows/coverage.yml
 .github/workflows/detect-internal-config-changes.yml
 .github/workflows/dockers-agent-faiss-image.yml
+.github/workflows/dockers-agent-image.yml
 .github/workflows/dockers-agent-ngt-image.yml
 .github/workflows/dockers-agent-sidecar-image.yml
 .github/workflows/dockers-benchmark-job-image.yml
@@ -167,8 +168,10 @@ apis/grpc/v1/mirror/mirror.pb.go
 apis/grpc/v1/mirror/mirror_vtproto.pb.go
 apis/grpc/v1/payload/interface.go
 apis/grpc/v1/payload/payload.pb.go
+apis/grpc/v1/payload/payload.pb.json.go
 apis/grpc/v1/payload/payload_vtproto.pb.go
 apis/grpc/v1/rpc/errdetails/error_details.pb.go
+apis/grpc/v1/rpc/errdetails/error_details.pb.json.go
 apis/grpc/v1/rpc/errdetails/error_details_vtproto.pb.go
 apis/grpc/v1/vald/filter.pb.go
 apis/grpc/v1/vald/filter_vtproto.pb.go
@@ -243,6 +246,7 @@ assets/docs/overview/search_flow.svg
 assets/docs/overview/update_flow.svg
 assets/docs/overview/upsert_flow.svg
 assets/docs/overview/vald_basic_architecture.svg
+assets/docs/performance/benchmark-grafana.png
 assets/docs/troubleshooting/provisioning_flow_chart.drawio
 assets/docs/troubleshooting/provisioning_flow_chart.svg
 assets/docs/tutorial/getstarted.svg
@@ -505,6 +509,8 @@ design/Vald Architecture Assets.drawio
 design/Vald Architecture Dataflow.drawio
 design/Vald Architecture Overview.drawio
 design/vald_architecture_overview.png
+dockers/agent/core/agent/Dockerfile
+dockers/agent/core/agent/README.md
 dockers/agent/core/faiss/Dockerfile
 dockers/agent/core/ngt/Dockerfile
 dockers/agent/core/ngt/README.md
@@ -1121,6 +1127,7 @@ internal/net/grpc/handler.go
 internal/net/grpc/health/health.go
 internal/net/grpc/health/health_test.go
 internal/net/grpc/interceptor.go
+internal/net/grpc/interceptor/client/metric/metric.go
 internal/net/grpc/interceptor/client/trace/trace.go
 internal/net/grpc/interceptor/server/logging/accesslog.go
 internal/net/grpc/interceptor/server/logging/accesslog_test.go
@@ -1814,6 +1821,11 @@ rust/Cargo.lock
 rust/Cargo.toml
 rust/bin/agent/Cargo.toml
 rust/bin/agent/src/main.rs
+rust/libs/ngt-rs/Cargo.toml
+rust/libs/ngt-rs/build.rs
+rust/libs/ngt-rs/src/input.cpp
+rust/libs/ngt-rs/src/input.h
+rust/libs/ngt-rs/src/lib.rs
 rust/libs/ngt/Cargo.toml
 rust/libs/ngt/src/lib.rs
 rust/libs/proto/Cargo.toml
@@ -1836,6 +1848,7 @@ rust/libs/proto/src/sidecar.v1.rs
 rust/libs/proto/src/sidecar.v1.tonic.rs
 rust/libs/proto/src/vald.v1.rs
 rust/libs/proto/src/vald.v1.tonic.rs
+rust/rust-toolchain.toml
 tests/chaos/chart/.helmignore
 tests/chaos/chart/Chart.yaml
 tests/chaos/chart/README.md
@@ -1881,6 +1894,7 @@ versions/OTEL_OPERATOR_VERSION
 versions/PROMETHEUS_STACK_VERSION
 versions/PROTOBUF_VERSION
 versions/REVIEWDOG_VERSION
+versions/RUST_VERSION
 versions/TELEPRESENCE_VERSION
 versions/VALDCLI_VERSION
 versions/VALD_VERSION

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,7 +22,8 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.2
+- Go Version: 1.22.3
+- Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.29.3
-- NGT Version: 2.2
+- Kubernetes Version: v1.30.0
+- NGT Version: 2.2.1

.github/ISSUE_TEMPLATE/security_issue_report.md

@@ -16,7 +16,8 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.2
+- Go Version: 1.22.3
+- Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.29.3
-- NGT Version: 2.2
+- Kubernetes Version: v1.30.0
+- NGT Version: 2.2.1

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,10 +16,11 @@
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.2
+- Go Version: 1.22.3
+- Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.29.3
-- NGT Version: 2.2
+- Kubernetes Version: v1.30.0
+- NGT Version: 2.2.1
 
 ### Checklist
 

.github/workflows/dockers-agent-image.yml

@@ -0,0 +1,74 @@
+#
+# Copyright (C) 2019-2024 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+name: "Build docker image: agent"
+on:
+  push:
+    branches:
+      - "main"
+      - "release/v*.*"
+      - "!release/v*.*.*"
+    tags:
+      - "*.*.*"
+      - "v*.*.*"
+      - "*.*.*-*"
+      - "v*.*.*-*"
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-agent-image.yml"
+      - "rust/Cargo.lock"
+      - "rust/Cargo.toml"
+      - "rust/bin/agent/**"
+      - "rust/libs/ngt/**"
+      - "rust/libs/ngt-rs/**"
+      - "rust/libs/proto/**"
+      - "dockers/agent/core/ngt-rust/Dockerfile"
+      - "versions/RUST_VERSION"
+      - "versions/NGT_VERSION"
+  pull_request:
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-agent-image.yml"
+      - "rust/Cargo.lock"
+      - "rust/Cargo.toml"
+      - "rust/bin/agent/**"
+      - "rust/libs/ngt/**"
+      - "rust/libs/ngt-rs/**"
+      - "rust/libs/proto/**"
+      - "dockers/agent/core/agent/Dockerfile"
+      - "versions/RUST_VERSION"
+      - "versions/NGT_VERSION"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/action.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-agent-image.yml"
+      - "rust/Cargo.lock"
+      - "rust/Cargo.toml"
+      - "rust/bin/agent/**"
+      - "rust/libs/ngt/**"
+      - "rust/libs/ngt-rs/**"
+      - "rust/libs/proto/**"
+      - "dockers/agent/core/agent/Dockerfile"
+      - "versions/RUST_VERSION"
+      - "versions/NGT_VERSION"
+jobs:
+  build:
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: agent
+    secrets: inherit

.github/workflows/update-pull-request-and-issue-template.yml

@@ -22,6 +22,7 @@ on:
       - "versions/GO_VERSION"
       - "versions/NGT_VERSION"
       - "versions/KUBECTL_VERSION"
+      - "versions/RUST_VERSION"
 jobs:
   dump-contexts-to-log:
     runs-on: ubuntu-latest

Makefile

@@ -24,6 +24,7 @@ GHCRORG                         = ghcr.io/$(ORG)/$(NAME)
 AGENT_NGT_IMAGE                 = $(NAME)-agent-ngt
 AGENT_FAISS_IMAGE               = $(NAME)-agent-faiss
 AGENT_SIDECAR_IMAGE             = $(NAME)-agent-sidecar
+AGENT_IMAGE                     = $(NAME)-agent
 CI_CONTAINER_IMAGE              = $(NAME)-ci-container
 DEV_CONTAINER_IMAGE             = $(NAME)-dev-container
 DISCOVERER_IMAGE                = $(NAME)-discoverer-k8s
@@ -58,9 +59,12 @@ GO_CLEAN_DEPS := true
 GOTEST_TIMEOUT = 30m
 CGO_ENABLED = 1
 
-RUST_HOME = /usr/local/lib/rust
-RUSTUP_HOME = $(RUST_HOME)/rustup
-CARGO_HOME = $(RUST_HOME)/cargo
+RUST_HOME ?= /usr/local/lib/rust
+RUSTUP_HOME ?= $(RUST_HOME)/rustup
+CARGO_HOME ?= $(RUST_HOME)/cargo
+RUST_VERSION := $(eval RUST_VERSION := $(shell cat versions/RUST_VERSION))$(RUST_VERSION)
+
+NPM_GLOBAL_PREFIX := $(eval NPM_GLOBAL_PREFIX := $(shell npm prefix --location=global))$(NPM_GLOBAL_PREFIX)
 
 NPM_GLOBAL_PREFIX := $(eval NPM_GLOBAL_PREFIX := $(shell npm prefix --location=global))$(NPM_GLOBAL_PREFIX)
 
@@ -138,14 +142,17 @@ ifeq ($(GOARCH),amd64)
 CFLAGS ?= -mno-avx512f -mno-avx512dq -mno-avx512cd -mno-avx512bw -mno-avx512vl
 CXXFLAGS ?= $(CFLAGS)
 EXTLDFLAGS ?= -m64
+NGT_EXTRA_FLAGS ?=
 else ifeq ($(GOARCH),arm64)
 CFLAGS ?=
 CXXFLAGS ?= $(CFLAGS)
 EXTLDFLAGS ?= -march=armv8-a
+NGT_EXTRA_FLAGS ?=
 else
 CFLAGS ?=
 CXXFLAGS ?= $(CFLAGS)
 EXTLDFLAGS ?=
+NGT_EXTRA_FLAGS ?=
 endif
 
 BENCH_DATASET_MD5S := $(eval BENCH_DATASET_MD5S := $(shell find $(BENCH_DATASET_MD5_DIR) -type f -regex ".*\.md5"))$(BENCH_DATASET_MD5S)
@@ -535,6 +542,11 @@ version/vald:
 version/go:
 	@echo $(GO_VERSION)
 
+.PHONY: version/rust
+## print rust version
+version/rust:
+	@echo $(RUST_VERSION)
+
 .PHONY: version/ngt
 ## print NGT version
 version/ngt:
@@ -571,7 +583,7 @@ ngt/install: /usr/local/include/NGT/Capi.h
 /usr/local/include/NGT/Capi.h:
 	git clone --depth 1 --branch v$(NGT_VERSION) https://github.com/yahoojapan/NGT $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	cd $(TEMP_DIR)/NGT-$(NGT_VERSION) && \
-		cmake -DCMAKE_C_FLAGS="$(CFLAGS)" -DCMAKE_CXX_FLAGS="$(CXXFLAGS)" .
+		cmake -DCMAKE_C_FLAGS="$(CFLAGS)" -DCMAKE_CXX_FLAGS="$(CXXFLAGS)" "$(NGT_EXTRA_FLAGS)" .
 	make -j -C $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	make install -C $(TEMP_DIR)/NGT-$(NGT_VERSION)
 	cd $(ROOTDIR)
@@ -582,7 +594,7 @@ ngt/install: /usr/local/include/NGT/Capi.h
 ## install Faiss
 faiss/install: /usr/local/lib/libfaiss.so
 /usr/local/lib/libfaiss.so:
-	curl -LO https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz
+	curl -fsSLO https://github.com/facebookresearch/faiss/archive/v$(FAISS_VERSION).tar.gz
 	tar zxf v$(FAISS_VERSION).tar.gz -C $(TEMP_DIR)/
 	cd $(TEMP_DIR)/faiss-$(FAISS_VERSION) && \
 		cmake -DFAISS_ENABLE_GPU=OFF -DFAISS_ENABLE_PYTHON=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -B build . && \

Makefile.d/build.mk

@@ -95,6 +95,12 @@ cmd/tools/benchmark/operator/operator:
 	$(eval CGO_ENABLED = 0)
 	$(call go-build,tools/benchmark/operator,,-static,,,$@)
 
+rust/target/release/agent:
+	pushd rust && cargo build -p agent --release && popd
+
+rust/target/debug/agent:
+	pushd rust && cargo build -p agent && popd
+
 .PHONY: binary/build/zip
 ## build all binaries and zip them
 binary/build/zip: \

Makefile.d/client.mk

@@ -21,14 +21,14 @@ valdcli/install: $(BINDIR)/valdcli
 ifeq ($(UNAME),Darwin)
 $(BINDIR)/valdcli:
 	mkdir -p $(BINDIR)
-	curl -LO https://github.com/rinx/vald-client-clj/releases/download/$(VALDCLI_VERSION)/valdcli-macos.zip
+	curl -fsSLO https://github.com/rinx/vald-client-clj/releases/download/$(VALDCLI_VERSION)/valdcli-macos.zip
 	unzip valdcli-macos.zip
 	rm -f valdcli-macos.zip
 	mv valdcli $(BINDIR)/valdcli
 else
 $(BINDIR)/valdcli:
 	mkdir -p $(BINDIR)
-	curl -LO https://github.com/rinx/vald-client-clj/releases/download/$(VALDCLI_VERSION)/valdcli-linux-static.zip
+	curl -fsSLO https://github.com/rinx/vald-client-clj/releases/download/$(VALDCLI_VERSION)/valdcli-linux-static.zip
 	unzip valdcli-linux-static.zip
 	rm -f valdcli-linux-static.zip
 	mv valdcli $(BINDIR)/valdcli