diff --git a/apis/docs/v1/docs.md b/apis/docs/v1/docs.md
index dd7ce8d7d8..fdf9b239fc 100644
--- a/apis/docs/v1/docs.md
+++ b/apis/docs/v1/docs.md
@@ -92,7 +92,6 @@
- [Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest)
- [Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest)
- [Upsert.Request](#payload-v1-Upsert-Request)
-
- [Remove.Timestamp.Operator](#payload-v1-Remove-Timestamp-Operator)
- [Search.AggregationAlgorithm](#payload-v1-Search-AggregationAlgorithm)
@@ -102,14 +101,14 @@
- [Insert](#vald-v1-Insert)
- [apis/proto/v1/vald/object.proto](#apis_proto_v1_vald_object-proto)
- [Object](#vald-v1-Object)
+- [apis/proto/v1/vald/remove.proto](#apis_proto_v1_vald_remove-proto)
+ - [Remove](#vald-v1-Remove)
- [apis/proto/v1/vald/search.proto](#apis_proto_v1_vald_search-proto)
- [Search](#vald-v1-Search)
- [apis/proto/v1/vald/update.proto](#apis_proto_v1_vald_update-proto)
- [Update](#vald-v1-Update)
- [apis/proto/v1/vald/upsert.proto](#apis_proto_v1_vald_upsert-proto)
- [Upsert](#vald-v1-Upsert)
-- [apis/proto/v1/vald/remove.proto](#apis_proto_v1_vald_remove-proto)
- - [Remove](#vald-v1-Remove)
- [Scalar Value Types](#scalar-value-types)
@@ -1054,6 +1053,25 @@ Object service provides ways to fetch indexed vectors.
| StreamGetObject | [.payload.v1.Object.VectorRequest](#payload-v1-Object-VectorRequest) stream | [.payload.v1.Object.StreamVector](#payload-v1-Object-StreamVector) stream | A method to fetch vectors by bidirectional streaming. |
| StreamListObject | [.payload.v1.Object.List.Request](#payload-v1-Object-List-Request) | [.payload.v1.Object.List.Response](#payload-v1-Object-List-Response) stream | A method to get all the vectors with server streaming |
+
+
+Top
+
+## apis/proto/v1/vald/remove.proto
+
+
+
+### Remove
+
+Remove service provides ways to remove indexed vectors.
+
+| Method Name | Request Type | Response Type | Description |
+| ----------------- | -------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- |
+| Remove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to remove an indexed vector. |
+| RemoveByTimestamp | [.payload.v1.Remove.TimestampRequest](#payload-v1-Remove-TimestampRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove an indexed vector based on timestamp. |
+| StreamRemove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to remove multiple indexed vectors by bidirectional streaming. |
+| MultiRemove | [.payload.v1.Remove.MultiRequest](#payload-v1-Remove-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove multiple indexed vectors in a single request. |
+
Top
@@ -1117,25 +1135,6 @@ Upsert service provides ways to insert/update vectors.
| StreamUpsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to insert/update multiple vectors by bidirectional streaming. |
| MultiUpsert | [.payload.v1.Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert/update multiple vectors in a single request. |
-
-
-Top
-
-## apis/proto/v1/vald/remove.proto
-
-
-
-### Remove
-
-Remove service provides ways to remove indexed vectors.
-
-| Method Name | Request Type | Response Type | Description |
-| ----------------- | -------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- |
-| Remove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to remove an indexed vector. |
-| RemoveByTimestamp | [.payload.v1.Remove.TimestampRequest](#payload-v1-Remove-TimestampRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove an indexed vector based on timestamp. |
-| StreamRemove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to remove multiple indexed vectors by bidirectional streaming. |
-| MultiRemove | [.payload.v1.Remove.MultiRequest](#payload-v1-Remove-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove multiple indexed vectors in a single request. |
-
## Scalar Value Types
| .proto Type | Notes | C++ | Java | Python | Go | C# | PHP | Ruby |
diff --git a/apis/grpc/v1/agent/core/agent.pb.go b/apis/grpc/v1/agent/core/agent.pb.go
index 291244328e..ae1691e36a 100644
--- a/apis/grpc/v1/agent/core/agent.pb.go
+++ b/apis/grpc/v1/agent/core/agent.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/agent/core/agent.proto
package core
diff --git a/apis/grpc/v1/agent/sidecar/sidecar.pb.go b/apis/grpc/v1/agent/sidecar/sidecar.pb.go
index 3a92b92784..9020a64633 100644
--- a/apis/grpc/v1/agent/sidecar/sidecar.pb.go
+++ b/apis/grpc/v1/agent/sidecar/sidecar.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/agent/sidecar/sidecar.proto
package sidecar
diff --git a/apis/grpc/v1/discoverer/discoverer.pb.go b/apis/grpc/v1/discoverer/discoverer.pb.go
index e29c580273..c527903b89 100644
--- a/apis/grpc/v1/discoverer/discoverer.pb.go
+++ b/apis/grpc/v1/discoverer/discoverer.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/discoverer/discoverer.proto
package discoverer
diff --git a/apis/grpc/v1/filter/egress/egress_filter.pb.go b/apis/grpc/v1/filter/egress/egress_filter.pb.go
index 4445aaf6b3..819bde27fb 100644
--- a/apis/grpc/v1/filter/egress/egress_filter.pb.go
+++ b/apis/grpc/v1/filter/egress/egress_filter.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/filter/egress/egress_filter.proto
package egress
diff --git a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go
index 31cebbe940..7750c68845 100644
--- a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go
+++ b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/filter/ingress/ingress_filter.proto
package ingress
diff --git a/apis/grpc/v1/manager/index/index_manager.pb.go b/apis/grpc/v1/manager/index/index_manager.pb.go
index 9635ee401a..02d723546e 100644
--- a/apis/grpc/v1/manager/index/index_manager.pb.go
+++ b/apis/grpc/v1/manager/index/index_manager.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/manager/index/index_manager.proto
package index
diff --git a/apis/grpc/v1/payload/payload.pb.go b/apis/grpc/v1/payload/payload.pb.go
index 89f77a395f..d598d92213 100644
--- a/apis/grpc/v1/payload/payload.pb.go
+++ b/apis/grpc/v1/payload/payload.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/payload/payload.proto
package payload
diff --git a/apis/grpc/v1/vald/filter.pb.go b/apis/grpc/v1/vald/filter.pb.go
index 85faf6c038..14a4e50b37 100644
--- a/apis/grpc/v1/vald/filter.pb.go
+++ b/apis/grpc/v1/vald/filter.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/vald/filter.proto
package vald
diff --git a/apis/grpc/v1/vald/insert.pb.go b/apis/grpc/v1/vald/insert.pb.go
index c53f018c13..aaf972ee05 100644
--- a/apis/grpc/v1/vald/insert.pb.go
+++ b/apis/grpc/v1/vald/insert.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/vald/insert.proto
package vald
diff --git a/apis/grpc/v1/vald/object.pb.go b/apis/grpc/v1/vald/object.pb.go
index e8b88fb2f8..0cefb0ac65 100644
--- a/apis/grpc/v1/vald/object.pb.go
+++ b/apis/grpc/v1/vald/object.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/vald/object.proto
package vald
diff --git a/apis/grpc/v1/vald/remove.pb.go b/apis/grpc/v1/vald/remove.pb.go
index 298d73e228..e4f4fcb9fa 100644
--- a/apis/grpc/v1/vald/remove.pb.go
+++ b/apis/grpc/v1/vald/remove.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/vald/remove.proto
package vald
diff --git a/apis/grpc/v1/vald/search.pb.go b/apis/grpc/v1/vald/search.pb.go
index b08189b9fc..5cb2ac80cd 100644
--- a/apis/grpc/v1/vald/search.pb.go
+++ b/apis/grpc/v1/vald/search.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/vald/search.proto
package vald
diff --git a/apis/grpc/v1/vald/update.pb.go b/apis/grpc/v1/vald/update.pb.go
index 1941f98082..a2ccc728f1 100644
--- a/apis/grpc/v1/vald/update.pb.go
+++ b/apis/grpc/v1/vald/update.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/vald/update.proto
package vald
diff --git a/apis/grpc/v1/vald/upsert.pb.go b/apis/grpc/v1/vald/upsert.pb.go
index 84c8f49918..573c23e587 100644
--- a/apis/grpc/v1/vald/upsert.pb.go
+++ b/apis/grpc/v1/vald/upsert.pb.go
@@ -17,7 +17,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.31.0
-// protoc v4.24.2
+// protoc v4.24.1
// source: apis/proto/v1/vald/upsert.proto
package vald
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json
index 49dc3382ec..ad756bf4e7 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json
@@ -388,11 +388,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json
index 43cde3b9f2..56456efacb 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json
@@ -156,11 +156,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json
index 7d76b24d4a..389fb067f1 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json
@@ -192,11 +192,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json
index 350b138215..f1c6892413 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json
@@ -177,11 +177,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json
index 3cd0d4b763..14c6982996 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json
@@ -374,11 +374,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json
index 9cbb4a2bf2..0301c85566 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json
@@ -156,11 +156,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json
index ada6e05746..7e1e5c24cd 100644
--- a/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json
+++ b/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json
@@ -156,11 +156,11 @@
"code": {
"type": "integer",
"format": "int32",
- "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]."
+ "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]."
},
"message": {
"type": "string",
- "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client."
+ "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client."
},
"details": {
"type": "array",
diff --git a/go.mod b/go.mod
index a627c2108f..962e292262 100755
--- a/go.mod
+++ b/go.mod
@@ -43,7 +43,7 @@ replace (
github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b
github.com/antihax/optional => github.com/antihax/optional v1.0.0
github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
- github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.334
+ github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.45.0
github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.21.0
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13
github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.37
@@ -120,7 +120,7 @@ replace (
github.com/go-playground/assert/v2 => github.com/go-playground/assert/v2 v2.2.0
github.com/go-playground/locales => github.com/go-playground/locales v0.14.1
github.com/go-playground/universal-translator => github.com/go-playground/universal-translator v0.18.1
- github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 v10.15.2
+ github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 v10.15.3
github.com/go-redis/redis/v8 => github.com/go-redis/redis/v8 v8.11.5
github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.7.1
github.com/go-task/slim-sprig => github.com/go-task/slim-sprig v2.20.0+incompatible
diff --git a/go.sum b/go.sum
index 4e229bbe98..9c36006c40 100644
--- a/go.sum
+++ b/go.sum
@@ -175,8 +175,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg=
github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
-github.com/aws/aws-sdk-go v1.44.334 h1:h2bdbGb//fez6Sv6PaYv868s9liDeoYM6hYsAqTB4MU=
-github.com/aws/aws-sdk-go v1.44.334/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.45.0 h1:qoVOQHuLacxJMO71T49KeE70zm+Tk3vtrl7XO4VUPZc=
+github.com/aws/aws-sdk-go v1.45.0/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0=
github.com/aws/aws-sdk-go-v2/config v1.18.37 h1:RNAfbPqw1CstCooHaTPhScz7z1PyocQj0UL+l95CgzI=
diff --git a/internal/core/algorithm/ngt/ngt.go b/internal/core/algorithm/ngt/ngt.go
index 80156eb208..7c277d72be 100644
--- a/internal/core/algorithm/ngt/ngt.go
+++ b/internal/core/algorithm/ngt/ngt.go
@@ -101,6 +101,7 @@ type (
index C.NGTIndex
ospace C.NGTObjectSpace
mu *sync.RWMutex
+ cmu *sync.RWMutex
}
)
@@ -221,6 +222,7 @@ func gen(isLoad bool, opts ...Option) (NGT, error) {
err error
)
n.mu = new(sync.RWMutex)
+ n.cmu = new(sync.RWMutex)
defer func() {
if err != nil {
@@ -387,7 +389,7 @@ func (n *ngt) Search(ctx context.Context, vec []float32, size int, epsilon, radi
radius = n.radius
}
- n.mu.RLock()
+ n.rLock(true)
ret := C.ngt_search_index_as_float(
n.index,
(*C.float)(&vec[0]),
@@ -400,10 +402,10 @@ func (n *ngt) Search(ctx context.Context, vec []float32, size int, epsilon, radi
if ret == ErrorCode {
ne := ebuf
- n.mu.RUnlock()
+ n.rUnlock(true)
return nil, n.newGoError(ne)
}
- n.mu.RUnlock()
+ n.rUnlock(true)
rsize := int(C.ngt_get_result_size(results, ebuf))
if rsize <= 0 {
@@ -448,7 +450,7 @@ func (n *ngt) LinearSearch(vec []float32, size int) (result []SearchResult, err
return nil, n.newGoError(ebuf)
}
- n.mu.RLock()
+ n.rLock(true)
ret := C.ngt_linear_search_index_as_float(
n.index,
(*C.float)(&vec[0]),
@@ -460,10 +462,10 @@ func (n *ngt) LinearSearch(vec []float32, size int) (result []SearchResult, err
if ret == ErrorCode {
ne := ebuf
- n.mu.RUnlock()
+ n.rUnlock(true)
return nil, n.newGoError(ne)
}
- n.mu.RUnlock()
+ n.rUnlock(true)
rsize := int(C.ngt_get_result_size(results, ebuf))
if rsize <= 0 {
@@ -498,9 +500,9 @@ func (n *ngt) Insert(vec []float32) (id uint, err error) {
}
ebuf := n.GetErrorBuffer()
- n.mu.Lock()
+ n.lock(true)
id = uint(C.ngt_insert_index_as_float(n.index, (*C.float)(&vec[0]), C.uint32_t(n.dimension), ebuf))
- n.mu.Unlock()
+ n.unlock(true)
if id == 0 {
return 0, n.newGoError(ebuf)
}
@@ -598,9 +600,9 @@ func (n *ngt) CreateIndex(poolSize uint32) error {
poolSize = n.poolSize
}
ebuf := n.GetErrorBuffer()
- n.mu.Lock()
+ n.lock(true)
ret := C.ngt_create_index(n.index, C.uint32_t(poolSize), ebuf)
- n.mu.Unlock()
+ n.unlock(true)
if ret == ErrorCode {
return n.newGoError(ebuf)
}
@@ -615,9 +617,9 @@ func (n *ngt) SaveIndex() error {
path := C.CString(n.idxPath)
defer C.free(unsafe.Pointer(path))
ebuf := n.GetErrorBuffer()
- n.mu.Lock()
+ n.rLock(true)
ret := C.ngt_save_index(n.index, path, ebuf)
- n.mu.Unlock()
+ n.rUnlock(true)
if ret == ErrorCode {
return n.newGoError(ebuf)
}
@@ -633,9 +635,9 @@ func (n *ngt) SaveIndexWithPath(idxPath string) error {
path := C.CString(idxPath)
defer C.free(unsafe.Pointer(path))
ebuf := n.GetErrorBuffer()
- n.mu.Lock()
+ n.rLock(true)
ret := C.ngt_save_index(n.index, path, ebuf)
- n.mu.Unlock()
+ n.rUnlock(true)
if ret == ErrorCode {
return n.newGoError(ebuf)
}
@@ -648,9 +650,9 @@ func (n *ngt) SaveIndexWithPath(idxPath string) error {
// Remove removes from NGT index.
func (n *ngt) Remove(id uint) error {
ebuf := n.GetErrorBuffer()
- n.mu.Lock()
+ n.lock(true)
ret := C.ngt_remove_index(n.index, C.ObjectID(id), ebuf)
- n.mu.Unlock()
+ n.unlock(true)
if ret == ErrorCode {
return n.newGoError(ebuf)
}
@@ -676,17 +678,17 @@ func (n *ngt) GetVector(id uint) (ret []float32, err error) {
ebuf := n.GetErrorBuffer()
switch n.objectType {
case Float:
- n.mu.RLock()
+ n.rLock(false)
results := C.ngt_get_object_as_float(n.ospace, C.ObjectID(id), ebuf)
- n.mu.RUnlock()
+ n.rUnlock(false)
if results == nil {
return nil, n.newGoError(ebuf)
}
ret = (*[algorithm.MaximumVectorDimensionSize]float32)(unsafe.Pointer(results))[:dimension:dimension]
case HalfFloat:
- n.mu.RLock()
+ n.rLock(false)
results := C.ngt_get_allocated_object_as_float(n.ospace, C.ObjectID(id), ebuf)
- n.mu.RUnlock()
+ n.rUnlock(false)
defer C.free(unsafe.Pointer(results))
if results == nil {
return nil, n.newGoError(ebuf)
@@ -696,9 +698,9 @@ func (n *ngt) GetVector(id uint) (ret []float32, err error) {
ret[i] = elem
}
case Uint8:
- n.mu.RLock()
+ n.rLock(false)
results := C.ngt_get_object_as_integer(n.ospace, C.ObjectID(id), ebuf)
- n.mu.RUnlock()
+ n.rUnlock(false)
if results == nil {
return nil, n.newGoError(ebuf)
}
@@ -747,3 +749,31 @@ func (n *ngt) GetErrorBuffer() (ebuf C.NGTError) {
func (n *ngt) PutErrorBuffer(ebuf C.NGTError) {
n.epool.Put(ebuf)
}
+
+func (n *ngt) lock(cLock bool) {
+ if cLock {
+ n.cmu.Lock()
+ }
+ n.mu.Lock()
+}
+
+func (n *ngt) unlock(cLock bool) {
+ n.mu.Unlock()
+ if cLock {
+ n.cmu.Unlock()
+ }
+}
+
+func (n *ngt) rLock(cLock bool) {
+ if cLock {
+ n.cmu.RLock()
+ }
+ n.mu.RLock()
+}
+
+func (n *ngt) rUnlock(cLock bool) {
+ n.mu.RUnlock()
+ if cLock {
+ n.cmu.RUnlock()
+ }
+}
diff --git a/internal/core/algorithm/ngt/ngt_test.go b/internal/core/algorithm/ngt/ngt_test.go
index 0e40fa3eca..2cc997f52f 100644
--- a/internal/core/algorithm/ngt/ngt_test.go
+++ b/internal/core/algorithm/ngt/ngt_test.go
@@ -139,6 +139,7 @@ func TestNew(t *testing.T) {
bulkInsertChunkSize: 100,
objectType: Float,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
},
comparators: append(ngtComparator, comparator.CompareField("idxPath", comparator.Comparer(func(s1, s2 string) bool {
@@ -164,6 +165,7 @@ func TestNew(t *testing.T) {
bulkInsertChunkSize: 100,
objectType: Float,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
},
}
@@ -188,6 +190,7 @@ func TestNew(t *testing.T) {
bulkInsertChunkSize: 100,
objectType: Uint8,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
},
}
@@ -316,6 +319,7 @@ func TestLoad(t *testing.T) {
bulkInsertChunkSize: 100,
objectType: Uint8,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
},
checkFunc: func(ctx context.Context, w want, n NGT, e error) error {
@@ -381,6 +385,7 @@ func TestLoad(t *testing.T) {
bulkInsertChunkSize: 100,
objectType: Uint8,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
},
checkFunc: func(ctx context.Context, w want, n NGT, e error) error {
@@ -446,6 +451,7 @@ func TestLoad(t *testing.T) {
bulkInsertChunkSize: 100,
objectType: Float,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
},
checkFunc: func(ctx context.Context, w want, n NGT, e error) error {
@@ -511,6 +517,7 @@ func TestLoad(t *testing.T) {
bulkInsertChunkSize: 100,
objectType: Float,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
},
checkFunc: func(ctx context.Context, w want, n NGT, e error) error {
@@ -706,6 +713,7 @@ func Test_gen(t *testing.T) {
bulkInsertChunkSize: 100,
objectType: Float,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
},
comparators: append(ngtComparator, comparator.CompareField("idxPath", comparator.Comparer(func(s1, s2 string) bool {
@@ -752,6 +760,7 @@ func Test_gen(t *testing.T) {
bulkInsertChunkSize: 100,
objectType: Uint8,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
},
checkFunc: func(ctx context.Context, w want, n NGT, e error, comparators ...comparator.Option) error {
@@ -1094,6 +1103,7 @@ func Test_ngt_open(t *testing.T) {
epsilon float32
poolSize uint32
mu *sync.RWMutex
+ cmu *sync.RWMutex
}
type want struct {
err error
@@ -1141,6 +1151,7 @@ func Test_ngt_open(t *testing.T) {
dimension: 9,
objectType: Float,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
beforeFunc: func(t *testing.T, fields fields) {
t.Helper()
@@ -1175,6 +1186,7 @@ func Test_ngt_open(t *testing.T) {
dimension: 9,
objectType: Float,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
want: want{
err: errors.ErrIndexFileNotFound,
@@ -1188,6 +1200,7 @@ func Test_ngt_open(t *testing.T) {
dimension: 9,
objectType: Float,
mu: &sync.RWMutex{},
+ cmu: &sync.RWMutex{},
},
beforeFunc: func(t *testing.T, fields fields) {
t.Helper()
diff --git a/pkg/agent/core/ngt/handler/grpc/object.go b/pkg/agent/core/ngt/handler/grpc/object.go
index 208b48bf91..b2e7a548b1 100644
--- a/pkg/agent/core/ngt/handler/grpc/object.go
+++ b/pkg/agent/core/ngt/handler/grpc/object.go
@@ -16,7 +16,6 @@ package grpc
import (
"context"
"fmt"
- "sync"
"github.com/vdaas/vald/apis/grpc/v1/payload"
"github.com/vdaas/vald/apis/grpc/v1/vald"
@@ -27,6 +26,7 @@ import (
"github.com/vdaas/vald/internal/net/grpc/errdetails"
"github.com/vdaas/vald/internal/net/grpc/status"
"github.com/vdaas/vald/internal/observability/trace"
+ "github.com/vdaas/vald/internal/sync"
)
func (s *server) Exists(ctx context.Context, uid *payload.Object_ID) (res *payload.Object_ID, err error) {
diff --git a/versions/NGT_VERSION b/versions/NGT_VERSION
index eca07e4c1a..ac2cdeba01 100644
--- a/versions/NGT_VERSION
+++ b/versions/NGT_VERSION
@@ -1 +1 @@
-2.1.2
+2.1.3