refactor index manager service add index service API to expose index …

…informations

Signed-off-by: kpango <[email protected]>

.gitfiles

@@ -161,8 +161,6 @@ apis/grpc/v1/filter/egress/egress_filter.pb.go
 apis/grpc/v1/filter/egress/egress_filter_vtproto.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter.pb.go
 apis/grpc/v1/filter/ingress/ingress_filter_vtproto.pb.go
-apis/grpc/v1/manager/index/index_manager.pb.go
-apis/grpc/v1/manager/index/index_manager_vtproto.pb.go
 apis/grpc/v1/mirror/mirror.go
 apis/grpc/v1/mirror/mirror.pb.go
 apis/grpc/v1/mirror/mirror_vtproto.pb.go
@@ -177,6 +175,8 @@ apis/grpc/v1/vald/filter.pb.go
 apis/grpc/v1/vald/filter_vtproto.pb.go
 apis/grpc/v1/vald/flush.pb.go
 apis/grpc/v1/vald/flush_vtproto.pb.go
+apis/grpc/v1/vald/index.pb.go
+apis/grpc/v1/vald/index_vtproto.pb.go
 apis/grpc/v1/vald/insert.pb.go
 apis/grpc/v1/vald/insert_vtproto.pb.go
 apis/grpc/v1/vald/object.pb.go
@@ -197,12 +197,12 @@ apis/proto/v1/agent/sidecar/sidecar.proto
 apis/proto/v1/discoverer/discoverer.proto
 apis/proto/v1/filter/egress/egress_filter.proto
 apis/proto/v1/filter/ingress/ingress_filter.proto
-apis/proto/v1/manager/index/index_manager.proto
 apis/proto/v1/mirror/mirror.proto
 apis/proto/v1/payload/payload.proto
 apis/proto/v1/rpc/errdetails/error_details.proto
 apis/proto/v1/vald/filter.proto
 apis/proto/v1/vald/flush.proto
+apis/proto/v1/vald/index.proto
 apis/proto/v1/vald/insert.proto
 apis/proto/v1/vald/object.proto
 apis/proto/v1/vald/remove.proto
@@ -214,12 +214,12 @@ apis/swagger/v1/agent/sidecar/sidecar.swagger.json
 apis/swagger/v1/discoverer/discoverer.swagger.json
 apis/swagger/v1/filter/egress/egress_filter.swagger.json
 apis/swagger/v1/filter/ingress/ingress_filter.swagger.json
-apis/swagger/v1/manager/index/index_manager.swagger.json
 apis/swagger/v1/mirror/mirror.swagger.json
 apis/swagger/v1/payload/payload.swagger.json
 apis/swagger/v1/rpc/errdetails/error_details.swagger.json
 apis/swagger/v1/vald/filter.swagger.json
 apis/swagger/v1/vald/flush.swagger.json
+apis/swagger/v1/vald/index.swagger.json
 apis/swagger/v1/vald/insert.swagger.json
 apis/swagger/v1/vald/object.swagger.json
 apis/swagger/v1/vald/remove.swagger.json
@@ -228,6 +228,7 @@ apis/swagger/v1/vald/update.swagger.json
 apis/swagger/v1/vald/upsert.swagger.json
 assets/docs/guides/observability-configuration/architecture.png
 assets/docs/guides/operations/grafana-example.png
+assets/docs/guides/read-replica-and-rotator/architecture.png
 assets/docs/overview/component/agent/agent.drawio
 assets/docs/overview/component/agent/ngt.png
 assets/docs/overview/component/agent/sidecar_backup.png
@@ -556,6 +557,7 @@ docs/api/upsert.md
 docs/contributing/coding-style.md
 docs/contributing/contributing-guide.md
 docs/contributing/development.md
+docs/contributing/reviewer-guideline.md
 docs/contributing/unit-test-guideline.md
 docs/overview/about-vald.md
 docs/overview/architecture.md
@@ -595,6 +597,7 @@ docs/user-guides/mirroring-configuration.md
 docs/user-guides/network-policy.md
 docs/user-guides/observability-configuration.md
 docs/user-guides/operations.md
+docs/user-guides/read-replica-and-rotator.md
 docs/user-guides/sdks.md
 docs/user-guides/upgrade-cluster.md
 example/client/agent/main.go
@@ -1820,6 +1823,14 @@ renovate.json
 rust/Cargo.lock
 rust/Cargo.toml
 rust/bin/agent/Cargo.toml
+rust/bin/agent/src/handler.rs
+rust/bin/agent/src/handler/common.rs
+rust/bin/agent/src/handler/index.rs
+rust/bin/agent/src/handler/insert.rs
+rust/bin/agent/src/handler/remove.rs
+rust/bin/agent/src/handler/search.rs
+rust/bin/agent/src/handler/update.rs
+rust/bin/agent/src/handler/upsert.rs
 rust/bin/agent/src/main.rs
 rust/libs/ngt-rs/Cargo.toml
 rust/libs/ngt-rs/build.rs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,8 +22,8 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.3
+- Go Version: 1.22.4
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.30.0
-- NGT Version: 2.2.1
+- Kubernetes Version: v1.30.2
+- NGT Version: 2.2.2

.github/ISSUE_TEMPLATE/security_issue_report.md

@@ -16,8 +16,8 @@ assignees: ""
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.3
+- Go Version: 1.22.4
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.30.0
-- NGT Version: 2.2.1
+- Kubernetes Version: v1.30.2
+- NGT Version: 2.2.2

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,11 +16,11 @@
 
 <!--- Please change the versions below along with your environment -->
 
-- Go Version: 1.22.3
+- Go Version: 1.22.4
 - Rust Version: 1.77.2
 - Docker Version: 20.10.8
-- Kubernetes Version: v1.30.0
-- NGT Version: 2.2.1
+- Kubernetes Version: v1.30.2
+- NGT Version: 2.2.2
 
 ### Checklist
 

.github/actions/docker-build/action.yaml

@@ -113,7 +113,11 @@ runs:
           REMOTE="true" \
           DOCKER="docker" \
           BUILDKIT_INLINE_CACHE=0 \
-          DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${LABEL_OPTS}  --label org.opencontainers.image.version=${PRIMARY_TAG} --label org.opencontainers.image.title=${TARGET}" \
+          DOCKER_OPTS="--platform ${PLATFORMS} --builder ${BUILDER} ${LABEL_OPTS} \
+          --label org.opencontainers.image.version=${PRIMARY_TAG} \
+          --label org.opencontainers.image.title=${TARGET} \
+          --label org.opencontainers.image.created=\"$(date --rfc-3339=ns)\" \
+          --label org.opencontainers.image.licenses=\"Apache 2.0\"" \
           EXTRA_ARGS="${EXTRA_TAGS}" \
           TAG="${PRIMARY_TAG}" \
           docker/build/${TARGET}

.github/workflows/release.yml

@@ -61,7 +61,7 @@ jobs:
           git push origin ${VERSION}
           echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT
       - name: Create release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         env:
           GITHUB_TOKEN: ${{ secrets.DISPATCH_TOKEN }}
         with:

Makefile.d/dependencies.mk

@@ -47,7 +47,8 @@ go/download:
 
 .PHONY: go/deps
 ## install Go package dependencies
-go/deps:
+go/deps: \
+	update/go
 	sed -i "3s/go [0-9]\+\.[0-9]\+\(\.[0-9]\+\)\?/go $(GO_VERSION)/g" $(ROOTDIR)/hack/go.mod.default
 	if $(GO_CLEAN_DEPS); then \
         	rm -rf $(ROOTDIR)/vendor \

Makefile.d/docker.mk

@@ -46,7 +46,9 @@ ifeq ($(REMOTE),true)
 	@echo "starting remote build for $(IMAGE):$(TAG)"
 	DOCKER_BUILDKIT=1 $(DOCKER) buildx build \
 		$(DOCKER_OPTS) \
+		--cache-to type=gha,scope=$(TAG)-buildcache,mode=max \
 		--cache-to type=registry,ref=$(GHCRORG)/$(IMAGE):$(TAG)-buildcache,mode=max \
+		--cache-from type=gha,scope=$(TAG)-buildcache \
 		--cache-from type=registry,ref=$(GHCRORG)/$(IMAGE):$(TAG)-buildcache \
 		--build-arg BUILDKIT_INLINE_CACHE=$(BUILDKIT_INLINE_CACHE) \
 		--build-arg GO_VERSION=$(GO_VERSION) \

Makefile.d/functions.mk

@@ -333,14 +333,12 @@ define gen-vald-crd
 endef
 
 define update-github-actions
-	@for ACTION_NAME in $1; do \
+	@set -e; for ACTION_NAME in $1; do \
 		if [ -n "$$ACTION_NAME" ] && [ "$$ACTION_NAME" != "security-and-quality" ]; then \
 			FILE_NAME=`echo $$ACTION_NAME | tr '/' '_' | tr '-' '_' | tr '[:lower:]' '[:upper:]'`; \
 			if [ -n "$$FILE_NAME" ]; then \
 				if [ "$$ACTION_NAME" = "aquasecurity/trivy-action" ] || [ "$$ACTION_NAME" = "machine-learning-apps/actions-chatops" ]; then \
 					VERSION="master"; \
-				elif [ "$$ACTION_NAME" = "softprops/action-gh-release" ]; then \
-					VERSION="1.0.0"; \
 				else \
 					REPO_NAME=`echo $$ACTION_NAME | cut -d'/' -f1-2`; \
 					VERSION=`curl -fsSL https://api.github.com/repos/$$REPO_NAME/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' | sed 's/v//g' | sed -E 's/[^0-9.]+//g'`;\

apis/docs/v1/docs.md

@@ -23,6 +23,8 @@
   - [Info.IPs](#payload-v1-Info-IPs)
   - [Info.Index](#payload-v1-Info-Index)
   - [Info.Index.Count](#payload-v1-Info-Index-Count)
+  - [Info.Index.Detail](#payload-v1-Info-Index-Detail)
+  - [Info.Index.Detail.CountsEntry](#payload-v1-Info-Index-Detail-CountsEntry)
   - [Info.Index.UUID](#payload-v1-Info-Index-UUID)
   - [Info.Index.UUID.Committed](#payload-v1-Info-Index-UUID-Committed)
   - [Info.Index.UUID.Uncommitted](#payload-v1-Info-Index-UUID-Uncommitted)
@@ -107,8 +109,6 @@
   - [Filter](#filter-egress-v1-Filter)
 - [v1/filter/ingress/ingress_filter.proto](#v1_filter_ingress_ingress_filter-proto)
   - [Filter](#filter-ingress-v1-Filter)
-- [v1/manager/index/index_manager.proto](#v1_manager_index_index_manager-proto)
-  - [Index](#manager-index-v1-Index)
 - [v1/mirror/mirror.proto](#v1_mirror_mirror-proto)
   - [Mirror](#mirror-v1-Mirror)
 - [v1/rpc/errdetails/error_details.proto](#v1_rpc_errdetails_error_details-proto)
@@ -131,6 +131,8 @@
   - [Filter](#vald-v1-Filter)
 - [v1/vald/flush.proto](#v1_vald_flush-proto)
   - [Flush](#vald-v1-Flush)
+- [v1/vald/index.proto](#v1_vald_index-proto)
+  - [Index](#vald-v1-Index)
 - [v1/vald/insert.proto](#v1_vald_insert-proto)
   - [Insert](#vald-v1-Insert)
 - [v1/vald/object.proto](#v1_vald_object-proto)
@@ -294,6 +296,27 @@ Represent the index count message.
 | indexing    | [bool](#bool)     |       | The indexing index count.    |
 | saving      | [bool](#bool)     |       | The saving index count.      |
 
+<a name="payload-v1-Info-Index-Detail"></a>
+
+### Info.Index.Detail
+
+Represent the index count for each Agents message.
+
+| Field       | Type                                                                       | Label    | Description                        |
+| ----------- | -------------------------------------------------------------------------- | -------- | ---------------------------------- |
+| counts      | [Info.Index.Detail.CountsEntry](#payload-v1-Info-Index-Detail-CountsEntry) | repeated | count infos for each agents        |
+| replica     | [uint32](#uint32)                                                          |          | index replica of vald cluster      |
+| live_agents | [uint32](#uint32)                                                          |          | live agent replica of vald cluster |
+
+<a name="payload-v1-Info-Index-Detail-CountsEntry"></a>
+
+### Info.Index.Detail.CountsEntry
+
+| Field | Type                                             | Label | Description |
+| ----- | ------------------------------------------------ | ----- | ----------- |
+| key   | [string](#string)                                |       |             |
+| value | [Info.Index.Count](#payload-v1-Info-Index-Count) |       |             |
+
 <a name="payload-v1-Info-Index-UUID"></a>
 
 ### Info.Index.UUID
@@ -1069,13 +1092,11 @@ AggregationAlgorithm is enum of each aggregation algorithms
 
 Represent the agent service.
 
-| Method Name        | Request Type                                                                     | Response Type                                                | Description                                                                                        |
-| ------------------ | -------------------------------------------------------------------------------- | ------------------------------------------------------------ | -------------------------------------------------------------------------------------------------- |
-| CreateIndex        | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty)                       | Represent the creating index RPC.                                                                  |
-| SaveIndex          | [.payload.v1.Empty](#payload-v1-Empty)                                           | [.payload.v1.Empty](#payload-v1-Empty)                       | Represent the saving index RPC.                                                                    |
-| CreateAndSaveIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty)                       | Represent the creating and saving index RPC.                                                       |
-| IndexInfo          | [.payload.v1.Empty](#payload-v1-Empty)                                           | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the agent index information.                                              |
-| GetTimestamp       | [.payload.v1.Object.GetTimestampRequest](#payload-v1-Object-GetTimestampRequest) | [.payload.v1.Object.Timestamp](#payload-v1-Object-Timestamp) | Represent the RPC to get the vector metadata. This RPC is mainly used for index correction process |
+| Method Name        | Request Type                                                                     | Response Type                          | Description                                  |
+| ------------------ | -------------------------------------------------------------------------------- | -------------------------------------- | -------------------------------------------- |
+| CreateIndex        | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating index RPC.            |
+| SaveIndex          | [.payload.v1.Empty](#payload-v1-Empty)                                           | [.payload.v1.Empty](#payload-v1-Empty) | Represent the saving index RPC.              |
+| CreateAndSaveIndex | [.payload.v1.Control.CreateIndexRequest](#payload-v1-Control-CreateIndexRequest) | [.payload.v1.Empty](#payload-v1-Empty) | Represent the creating and saving index RPC. |
 
 <a name="v1_agent_sidecar_sidecar-proto"></a>
 
@@ -1144,22 +1165,6 @@ Represent the ingress filter service.
 | GenVector    | [.payload.v1.Object.Blob](#payload-v1-Object-Blob)     | [.payload.v1.Object.Vector](#payload-v1-Object-Vector) | Represent the RPC to generate the vector. |
 | FilterVector | [.payload.v1.Object.Vector](#payload-v1-Object-Vector) | [.payload.v1.Object.Vector](#payload-v1-Object-Vector) | Represent the RPC to filter the vector.   |
 
-<a name="v1_manager_index_index_manager-proto"></a>
-
-<p align="right"><a href="#top">Top</a></p>
-
-## v1/manager/index/index_manager.proto
-
-<a name="manager-index-v1-Index"></a>
-
-### Index
-
-Represent the index manager service.
-
-| Method Name | Request Type                           | Response Type                                                | Description                                     |
-| ----------- | -------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------- |
-| IndexInfo   | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the index information. |
-
 <a name="v1_mirror_mirror-proto"></a>
 
 <p align="right"><a href="#top">Top</a></p>
@@ -1480,6 +1485,23 @@ Flush service provides ways to flush all indexed vectors.
 | ----------- | ------------------------------------------------------ | ------------------------------------------------------------ | ------------------------------------- |
 | Flush       | [.payload.v1.Flush.Request](#payload-v1-Flush-Request) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | A method to flush all indexed vector. |
 
+<a name="v1_vald_index-proto"></a>
+
+<p align="right"><a href="#top">Top</a></p>
+
+## v1/vald/index.proto
+
+<a name="vald-v1-Index"></a>
+
+### Index
+
+Represent the index manager service.
+
+| Method Name | Request Type                           | Response Type                                                  | Description                                                     |
+| ----------- | -------------------------------------- | -------------------------------------------------------------- | --------------------------------------------------------------- |
+| IndexInfo   | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count)   | Represent the RPC to get the index information.                 |
+| IndexDetail | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Detail](#payload-v1-Info-Index-Detail) | Represent the RPC to get the index information for each agents. |
+
 <a name="v1_vald_insert-proto"></a>
 
 <p align="right"><a href="#top">Top</a></p>
@@ -1510,12 +1532,13 @@ Insert service provides ways to add new vectors.
 
 Object service provides ways to fetch indexed vectors.
 
-| Method Name      | Request Type                                                                | Response Type                                                               | Description                                                 |
-| ---------------- | --------------------------------------------------------------------------- | --------------------------------------------------------------------------- | ----------------------------------------------------------- |
-| Exists           | [.payload.v1.Object.ID](#payload-v1-Object-ID)                              | [.payload.v1.Object.ID](#payload-v1-Object-ID)                              | A method to check whether a specified ID is indexed or not. |
-| GetObject        | [.payload.v1.Object.VectorRequest](#payload-v1-Object-VectorRequest)        | [.payload.v1.Object.Vector](#payload-v1-Object-Vector)                      | A method to fetch a vector.                                 |
-| StreamGetObject  | [.payload.v1.Object.VectorRequest](#payload-v1-Object-VectorRequest) stream | [.payload.v1.Object.StreamVector](#payload-v1-Object-StreamVector) stream   | A method to fetch vectors by bidirectional streaming.       |
-| StreamListObject | [.payload.v1.Object.List.Request](#payload-v1-Object-List-Request)          | [.payload.v1.Object.List.Response](#payload-v1-Object-List-Response) stream | A method to get all the vectors with server streaming       |
+| Method Name      | Request Type                                                                     | Response Type                                                               | Description                                                                                        |
+| ---------------- | -------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- |
+| Exists           | [.payload.v1.Object.ID](#payload-v1-Object-ID)                                   | [.payload.v1.Object.ID](#payload-v1-Object-ID)                              | A method to check whether a specified ID is indexed or not.                                        |
+| GetObject        | [.payload.v1.Object.VectorRequest](#payload-v1-Object-VectorRequest)             | [.payload.v1.Object.Vector](#payload-v1-Object-Vector)                      | A method to fetch a vector.                                                                        |
+| StreamGetObject  | [.payload.v1.Object.VectorRequest](#payload-v1-Object-VectorRequest) stream      | [.payload.v1.Object.StreamVector](#payload-v1-Object-StreamVector) stream   | A method to fetch vectors by bidirectional streaming.                                              |
+| StreamListObject | [.payload.v1.Object.List.Request](#payload-v1-Object-List-Request)               | [.payload.v1.Object.List.Response](#payload-v1-Object-List-Response) stream | A method to get all the vectors with server streaming                                              |
+| GetTimestamp     | [.payload.v1.Object.GetTimestampRequest](#payload-v1-Object-GetTimestampRequest) | [.payload.v1.Object.Timestamp](#payload-v1-Object-Timestamp)                | Represent the RPC to get the vector metadata. This RPC is mainly used for index correction process |
 
 <a name="v1_vald_remove-proto"></a>
 

apis/grpc/v1/agent/core/agent.go

@@ -21,6 +21,4 @@ const (
 	CreateIndexRPCName        = "CreateIndex"
 	SaveIndexRPCName          = "SaveIndex"
 	CreateAndSaveIndexRPCName = "CreateAndSaveIndex"
-	IndexInfoRPCName          = "IndexInfo"
-	GetTimestampRPCName       = "GetTimestamp"
 )