From 8ee1316af3478d0d3bff3a50c5f0056dc4a78b99 Mon Sep 17 00:00:00 2001 From: ykadowak Date: Thu, 30 Nov 2023 02:46:06 +0000 Subject: [PATCH] Revert "refactor internal/core/algorithm/ngt mutex lock timing (#2144)" This reverts commit 3320464cff01371c3cbf0ec36ddd9a1365535ee4. --- apis/docs/v1/docs.md | 43 +++++------ apis/grpc/v1/agent/core/agent.pb.go | 2 +- apis/grpc/v1/agent/sidecar/sidecar.pb.go | 2 +- apis/grpc/v1/discoverer/discoverer.pb.go | 2 +- .../grpc/v1/filter/egress/egress_filter.pb.go | 2 +- .../v1/filter/ingress/ingress_filter.pb.go | 2 +- .../grpc/v1/manager/index/index_manager.pb.go | 2 +- apis/grpc/v1/payload/payload.pb.go | 2 +- apis/grpc/v1/vald/filter.pb.go | 2 +- apis/grpc/v1/vald/insert.pb.go | 2 +- apis/grpc/v1/vald/object.pb.go | 2 +- apis/grpc/v1/vald/remove.pb.go | 2 +- apis/grpc/v1/vald/search.pb.go | 2 +- apis/grpc/v1/vald/update.pb.go | 2 +- apis/grpc/v1/vald/upsert.pb.go | 2 +- .../apis/proto/v1/vald/filter.swagger.json | 4 +- .../apis/proto/v1/vald/insert.swagger.json | 4 +- .../apis/proto/v1/vald/object.swagger.json | 4 +- .../apis/proto/v1/vald/remove.swagger.json | 4 +- .../apis/proto/v1/vald/search.swagger.json | 4 +- .../apis/proto/v1/vald/update.swagger.json | 4 +- .../apis/proto/v1/vald/upsert.swagger.json | 4 +- go.mod | 4 +- go.sum | 4 +- internal/core/algorithm/ngt/ngt.go | 74 ++++++------------- internal/core/algorithm/ngt/ngt_test.go | 13 ---- pkg/agent/core/ngt/handler/grpc/object.go | 2 +- versions/NGT_VERSION | 2 +- 28 files changed, 78 insertions(+), 120 deletions(-) diff --git a/apis/docs/v1/docs.md b/apis/docs/v1/docs.md index fdf9b239fc..dd7ce8d7d8 100644 --- a/apis/docs/v1/docs.md +++ b/apis/docs/v1/docs.md @@ -92,6 +92,7 @@ - [Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest) - [Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) - [Upsert.Request](#payload-v1-Upsert-Request) + - [Remove.Timestamp.Operator](#payload-v1-Remove-Timestamp-Operator) - [Search.AggregationAlgorithm](#payload-v1-Search-AggregationAlgorithm) @@ -101,14 +102,14 @@ - [Insert](#vald-v1-Insert) - [apis/proto/v1/vald/object.proto](#apis_proto_v1_vald_object-proto) - [Object](#vald-v1-Object) -- [apis/proto/v1/vald/remove.proto](#apis_proto_v1_vald_remove-proto) - - [Remove](#vald-v1-Remove) - [apis/proto/v1/vald/search.proto](#apis_proto_v1_vald_search-proto) - [Search](#vald-v1-Search) - [apis/proto/v1/vald/update.proto](#apis_proto_v1_vald_update-proto) - [Update](#vald-v1-Update) - [apis/proto/v1/vald/upsert.proto](#apis_proto_v1_vald_upsert-proto) - [Upsert](#vald-v1-Upsert) +- [apis/proto/v1/vald/remove.proto](#apis_proto_v1_vald_remove-proto) + - [Remove](#vald-v1-Remove) - [Scalar Value Types](#scalar-value-types) @@ -1053,25 +1054,6 @@ Object service provides ways to fetch indexed vectors. | StreamGetObject | [.payload.v1.Object.VectorRequest](#payload-v1-Object-VectorRequest) stream | [.payload.v1.Object.StreamVector](#payload-v1-Object-StreamVector) stream | A method to fetch vectors by bidirectional streaming. | | StreamListObject | [.payload.v1.Object.List.Request](#payload-v1-Object-List-Request) | [.payload.v1.Object.List.Response](#payload-v1-Object-List-Response) stream | A method to get all the vectors with server streaming | - - -

Top

- -## apis/proto/v1/vald/remove.proto - - - -### Remove - -Remove service provides ways to remove indexed vectors. - -| Method Name | Request Type | Response Type | Description | -| ----------------- | -------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- | -| Remove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to remove an indexed vector. | -| RemoveByTimestamp | [.payload.v1.Remove.TimestampRequest](#payload-v1-Remove-TimestampRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove an indexed vector based on timestamp. | -| StreamRemove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to remove multiple indexed vectors by bidirectional streaming. | -| MultiRemove | [.payload.v1.Remove.MultiRequest](#payload-v1-Remove-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove multiple indexed vectors in a single request. | -

Top

@@ -1135,6 +1117,25 @@ Upsert service provides ways to insert/update vectors. | StreamUpsert | [.payload.v1.Upsert.Request](#payload-v1-Upsert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to insert/update multiple vectors by bidirectional streaming. | | MultiUpsert | [.payload.v1.Upsert.MultiRequest](#payload-v1-Upsert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert/update multiple vectors in a single request. | + + +

Top

+ +## apis/proto/v1/vald/remove.proto + + + +### Remove + +Remove service provides ways to remove indexed vectors. + +| Method Name | Request Type | Response Type | Description | +| ----------------- | -------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| Remove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to remove an indexed vector. | +| RemoveByTimestamp | [.payload.v1.Remove.TimestampRequest](#payload-v1-Remove-TimestampRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove an indexed vector based on timestamp. | +| StreamRemove | [.payload.v1.Remove.Request](#payload-v1-Remove-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to remove multiple indexed vectors by bidirectional streaming. | +| MultiRemove | [.payload.v1.Remove.MultiRequest](#payload-v1-Remove-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to remove multiple indexed vectors in a single request. | + ## Scalar Value Types | .proto Type | Notes | C++ | Java | Python | Go | C# | PHP | Ruby | diff --git a/apis/grpc/v1/agent/core/agent.pb.go b/apis/grpc/v1/agent/core/agent.pb.go index ae1691e36a..291244328e 100644 --- a/apis/grpc/v1/agent/core/agent.pb.go +++ b/apis/grpc/v1/agent/core/agent.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/agent/core/agent.proto package core diff --git a/apis/grpc/v1/agent/sidecar/sidecar.pb.go b/apis/grpc/v1/agent/sidecar/sidecar.pb.go index 9020a64633..3a92b92784 100644 --- a/apis/grpc/v1/agent/sidecar/sidecar.pb.go +++ b/apis/grpc/v1/agent/sidecar/sidecar.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/agent/sidecar/sidecar.proto package sidecar diff --git a/apis/grpc/v1/discoverer/discoverer.pb.go b/apis/grpc/v1/discoverer/discoverer.pb.go index c527903b89..e29c580273 100644 --- a/apis/grpc/v1/discoverer/discoverer.pb.go +++ b/apis/grpc/v1/discoverer/discoverer.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/discoverer/discoverer.proto package discoverer diff --git a/apis/grpc/v1/filter/egress/egress_filter.pb.go b/apis/grpc/v1/filter/egress/egress_filter.pb.go index 819bde27fb..4445aaf6b3 100644 --- a/apis/grpc/v1/filter/egress/egress_filter.pb.go +++ b/apis/grpc/v1/filter/egress/egress_filter.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/filter/egress/egress_filter.proto package egress diff --git a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go index 7750c68845..31cebbe940 100644 --- a/apis/grpc/v1/filter/ingress/ingress_filter.pb.go +++ b/apis/grpc/v1/filter/ingress/ingress_filter.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/filter/ingress/ingress_filter.proto package ingress diff --git a/apis/grpc/v1/manager/index/index_manager.pb.go b/apis/grpc/v1/manager/index/index_manager.pb.go index 02d723546e..9635ee401a 100644 --- a/apis/grpc/v1/manager/index/index_manager.pb.go +++ b/apis/grpc/v1/manager/index/index_manager.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/manager/index/index_manager.proto package index diff --git a/apis/grpc/v1/payload/payload.pb.go b/apis/grpc/v1/payload/payload.pb.go index d598d92213..89f77a395f 100644 --- a/apis/grpc/v1/payload/payload.pb.go +++ b/apis/grpc/v1/payload/payload.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/payload/payload.proto package payload diff --git a/apis/grpc/v1/vald/filter.pb.go b/apis/grpc/v1/vald/filter.pb.go index 14a4e50b37..85faf6c038 100644 --- a/apis/grpc/v1/vald/filter.pb.go +++ b/apis/grpc/v1/vald/filter.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/vald/filter.proto package vald diff --git a/apis/grpc/v1/vald/insert.pb.go b/apis/grpc/v1/vald/insert.pb.go index aaf972ee05..c53f018c13 100644 --- a/apis/grpc/v1/vald/insert.pb.go +++ b/apis/grpc/v1/vald/insert.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/vald/insert.proto package vald diff --git a/apis/grpc/v1/vald/object.pb.go b/apis/grpc/v1/vald/object.pb.go index 0cefb0ac65..e8b88fb2f8 100644 --- a/apis/grpc/v1/vald/object.pb.go +++ b/apis/grpc/v1/vald/object.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/vald/object.proto package vald diff --git a/apis/grpc/v1/vald/remove.pb.go b/apis/grpc/v1/vald/remove.pb.go index e4f4fcb9fa..298d73e228 100644 --- a/apis/grpc/v1/vald/remove.pb.go +++ b/apis/grpc/v1/vald/remove.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/vald/remove.proto package vald diff --git a/apis/grpc/v1/vald/search.pb.go b/apis/grpc/v1/vald/search.pb.go index 5cb2ac80cd..b08189b9fc 100644 --- a/apis/grpc/v1/vald/search.pb.go +++ b/apis/grpc/v1/vald/search.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/vald/search.proto package vald diff --git a/apis/grpc/v1/vald/update.pb.go b/apis/grpc/v1/vald/update.pb.go index a2ccc728f1..1941f98082 100644 --- a/apis/grpc/v1/vald/update.pb.go +++ b/apis/grpc/v1/vald/update.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/vald/update.proto package vald diff --git a/apis/grpc/v1/vald/upsert.pb.go b/apis/grpc/v1/vald/upsert.pb.go index 573c23e587..84c8f49918 100644 --- a/apis/grpc/v1/vald/upsert.pb.go +++ b/apis/grpc/v1/vald/upsert.pb.go @@ -17,7 +17,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.31.0 -// protoc v4.24.1 +// protoc v4.24.2 // source: apis/proto/v1/vald/upsert.proto package vald diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json index ad756bf4e7..49dc3382ec 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/filter.swagger.json @@ -388,11 +388,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json index 56456efacb..43cde3b9f2 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/insert.swagger.json @@ -156,11 +156,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json index 389fb067f1..7d76b24d4a 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/object.swagger.json @@ -192,11 +192,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json index f1c6892413..350b138215 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/remove.swagger.json @@ -177,11 +177,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json index 14c6982996..3cd0d4b763 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/search.swagger.json @@ -374,11 +374,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json index 0301c85566..9cbb4a2bf2 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/update.swagger.json @@ -156,11 +156,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." }, "details": { "type": "array", diff --git a/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json b/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json index 7e1e5c24cd..ada6e05746 100644 --- a/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json +++ b/apis/swagger/v1/vald/apis/proto/v1/vald/upsert.swagger.json @@ -156,11 +156,11 @@ "code": { "type": "integer", "format": "int32", - "description": "The status code, which should be an enum value of [google.rpc.Code][google.rpc.Code]." + "description": "The status code, which should be an enum value of\n[google.rpc.Code][google.rpc.Code]." }, "message": { "type": "string", - "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized by the client." + "description": "A developer-facing error message, which should be in English. Any\nuser-facing error message should be localized and sent in the\n[google.rpc.Status.details][google.rpc.Status.details] field, or localized\nby the client." }, "details": { "type": "array", diff --git a/go.mod b/go.mod index 962e292262..a627c2108f 100755 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ replace ( github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b github.com/antihax/optional => github.com/antihax/optional v1.0.0 github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 - github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.45.0 + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.44.334 github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.37 @@ -120,7 +120,7 @@ replace ( github.com/go-playground/assert/v2 => github.com/go-playground/assert/v2 v2.2.0 github.com/go-playground/locales => github.com/go-playground/locales v0.14.1 github.com/go-playground/universal-translator => github.com/go-playground/universal-translator v0.18.1 - github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 v10.15.3 + github.com/go-playground/validator/v10 => github.com/go-playground/validator/v10 v10.15.2 github.com/go-redis/redis/v8 => github.com/go-redis/redis/v8 v8.11.5 github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.7.1 github.com/go-task/slim-sprig => github.com/go-task/slim-sprig v2.20.0+incompatible diff --git a/go.sum b/go.sum index 9c36006c40..4e229bbe98 100644 --- a/go.sum +++ b/go.sum @@ -175,8 +175,8 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/aws/aws-sdk-go v1.45.0 h1:qoVOQHuLacxJMO71T49KeE70zm+Tk3vtrl7XO4VUPZc= -github.com/aws/aws-sdk-go v1.45.0/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.334 h1:h2bdbGb//fez6Sv6PaYv868s9liDeoYM6hYsAqTB4MU= +github.com/aws/aws-sdk-go v1.44.334/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= github.com/aws/aws-sdk-go-v2/config v1.18.37 h1:RNAfbPqw1CstCooHaTPhScz7z1PyocQj0UL+l95CgzI= diff --git a/internal/core/algorithm/ngt/ngt.go b/internal/core/algorithm/ngt/ngt.go index 7c277d72be..80156eb208 100644 --- a/internal/core/algorithm/ngt/ngt.go +++ b/internal/core/algorithm/ngt/ngt.go @@ -101,7 +101,6 @@ type ( index C.NGTIndex ospace C.NGTObjectSpace mu *sync.RWMutex - cmu *sync.RWMutex } ) @@ -222,7 +221,6 @@ func gen(isLoad bool, opts ...Option) (NGT, error) { err error ) n.mu = new(sync.RWMutex) - n.cmu = new(sync.RWMutex) defer func() { if err != nil { @@ -389,7 +387,7 @@ func (n *ngt) Search(ctx context.Context, vec []float32, size int, epsilon, radi radius = n.radius } - n.rLock(true) + n.mu.RLock() ret := C.ngt_search_index_as_float( n.index, (*C.float)(&vec[0]), @@ -402,10 +400,10 @@ func (n *ngt) Search(ctx context.Context, vec []float32, size int, epsilon, radi if ret == ErrorCode { ne := ebuf - n.rUnlock(true) + n.mu.RUnlock() return nil, n.newGoError(ne) } - n.rUnlock(true) + n.mu.RUnlock() rsize := int(C.ngt_get_result_size(results, ebuf)) if rsize <= 0 { @@ -450,7 +448,7 @@ func (n *ngt) LinearSearch(vec []float32, size int) (result []SearchResult, err return nil, n.newGoError(ebuf) } - n.rLock(true) + n.mu.RLock() ret := C.ngt_linear_search_index_as_float( n.index, (*C.float)(&vec[0]), @@ -462,10 +460,10 @@ func (n *ngt) LinearSearch(vec []float32, size int) (result []SearchResult, err if ret == ErrorCode { ne := ebuf - n.rUnlock(true) + n.mu.RUnlock() return nil, n.newGoError(ne) } - n.rUnlock(true) + n.mu.RUnlock() rsize := int(C.ngt_get_result_size(results, ebuf)) if rsize <= 0 { @@ -500,9 +498,9 @@ func (n *ngt) Insert(vec []float32) (id uint, err error) { } ebuf := n.GetErrorBuffer() - n.lock(true) + n.mu.Lock() id = uint(C.ngt_insert_index_as_float(n.index, (*C.float)(&vec[0]), C.uint32_t(n.dimension), ebuf)) - n.unlock(true) + n.mu.Unlock() if id == 0 { return 0, n.newGoError(ebuf) } @@ -600,9 +598,9 @@ func (n *ngt) CreateIndex(poolSize uint32) error { poolSize = n.poolSize } ebuf := n.GetErrorBuffer() - n.lock(true) + n.mu.Lock() ret := C.ngt_create_index(n.index, C.uint32_t(poolSize), ebuf) - n.unlock(true) + n.mu.Unlock() if ret == ErrorCode { return n.newGoError(ebuf) } @@ -617,9 +615,9 @@ func (n *ngt) SaveIndex() error { path := C.CString(n.idxPath) defer C.free(unsafe.Pointer(path)) ebuf := n.GetErrorBuffer() - n.rLock(true) + n.mu.Lock() ret := C.ngt_save_index(n.index, path, ebuf) - n.rUnlock(true) + n.mu.Unlock() if ret == ErrorCode { return n.newGoError(ebuf) } @@ -635,9 +633,9 @@ func (n *ngt) SaveIndexWithPath(idxPath string) error { path := C.CString(idxPath) defer C.free(unsafe.Pointer(path)) ebuf := n.GetErrorBuffer() - n.rLock(true) + n.mu.Lock() ret := C.ngt_save_index(n.index, path, ebuf) - n.rUnlock(true) + n.mu.Unlock() if ret == ErrorCode { return n.newGoError(ebuf) } @@ -650,9 +648,9 @@ func (n *ngt) SaveIndexWithPath(idxPath string) error { // Remove removes from NGT index. func (n *ngt) Remove(id uint) error { ebuf := n.GetErrorBuffer() - n.lock(true) + n.mu.Lock() ret := C.ngt_remove_index(n.index, C.ObjectID(id), ebuf) - n.unlock(true) + n.mu.Unlock() if ret == ErrorCode { return n.newGoError(ebuf) } @@ -678,17 +676,17 @@ func (n *ngt) GetVector(id uint) (ret []float32, err error) { ebuf := n.GetErrorBuffer() switch n.objectType { case Float: - n.rLock(false) + n.mu.RLock() results := C.ngt_get_object_as_float(n.ospace, C.ObjectID(id), ebuf) - n.rUnlock(false) + n.mu.RUnlock() if results == nil { return nil, n.newGoError(ebuf) } ret = (*[algorithm.MaximumVectorDimensionSize]float32)(unsafe.Pointer(results))[:dimension:dimension] case HalfFloat: - n.rLock(false) + n.mu.RLock() results := C.ngt_get_allocated_object_as_float(n.ospace, C.ObjectID(id), ebuf) - n.rUnlock(false) + n.mu.RUnlock() defer C.free(unsafe.Pointer(results)) if results == nil { return nil, n.newGoError(ebuf) @@ -698,9 +696,9 @@ func (n *ngt) GetVector(id uint) (ret []float32, err error) { ret[i] = elem } case Uint8: - n.rLock(false) + n.mu.RLock() results := C.ngt_get_object_as_integer(n.ospace, C.ObjectID(id), ebuf) - n.rUnlock(false) + n.mu.RUnlock() if results == nil { return nil, n.newGoError(ebuf) } @@ -749,31 +747,3 @@ func (n *ngt) GetErrorBuffer() (ebuf C.NGTError) { func (n *ngt) PutErrorBuffer(ebuf C.NGTError) { n.epool.Put(ebuf) } - -func (n *ngt) lock(cLock bool) { - if cLock { - n.cmu.Lock() - } - n.mu.Lock() -} - -func (n *ngt) unlock(cLock bool) { - n.mu.Unlock() - if cLock { - n.cmu.Unlock() - } -} - -func (n *ngt) rLock(cLock bool) { - if cLock { - n.cmu.RLock() - } - n.mu.RLock() -} - -func (n *ngt) rUnlock(cLock bool) { - n.mu.RUnlock() - if cLock { - n.cmu.RUnlock() - } -} diff --git a/internal/core/algorithm/ngt/ngt_test.go b/internal/core/algorithm/ngt/ngt_test.go index 2cc997f52f..0e40fa3eca 100644 --- a/internal/core/algorithm/ngt/ngt_test.go +++ b/internal/core/algorithm/ngt/ngt_test.go @@ -139,7 +139,6 @@ func TestNew(t *testing.T) { bulkInsertChunkSize: 100, objectType: Float, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, }, comparators: append(ngtComparator, comparator.CompareField("idxPath", comparator.Comparer(func(s1, s2 string) bool { @@ -165,7 +164,6 @@ func TestNew(t *testing.T) { bulkInsertChunkSize: 100, objectType: Float, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, }, } @@ -190,7 +188,6 @@ func TestNew(t *testing.T) { bulkInsertChunkSize: 100, objectType: Uint8, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, }, } @@ -319,7 +316,6 @@ func TestLoad(t *testing.T) { bulkInsertChunkSize: 100, objectType: Uint8, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error) error { @@ -385,7 +381,6 @@ func TestLoad(t *testing.T) { bulkInsertChunkSize: 100, objectType: Uint8, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error) error { @@ -451,7 +446,6 @@ func TestLoad(t *testing.T) { bulkInsertChunkSize: 100, objectType: Float, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error) error { @@ -517,7 +511,6 @@ func TestLoad(t *testing.T) { bulkInsertChunkSize: 100, objectType: Float, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error) error { @@ -713,7 +706,6 @@ func Test_gen(t *testing.T) { bulkInsertChunkSize: 100, objectType: Float, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, }, comparators: append(ngtComparator, comparator.CompareField("idxPath", comparator.Comparer(func(s1, s2 string) bool { @@ -760,7 +752,6 @@ func Test_gen(t *testing.T) { bulkInsertChunkSize: 100, objectType: Uint8, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, }, checkFunc: func(ctx context.Context, w want, n NGT, e error, comparators ...comparator.Option) error { @@ -1103,7 +1094,6 @@ func Test_ngt_open(t *testing.T) { epsilon float32 poolSize uint32 mu *sync.RWMutex - cmu *sync.RWMutex } type want struct { err error @@ -1151,7 +1141,6 @@ func Test_ngt_open(t *testing.T) { dimension: 9, objectType: Float, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, beforeFunc: func(t *testing.T, fields fields) { t.Helper() @@ -1186,7 +1175,6 @@ func Test_ngt_open(t *testing.T) { dimension: 9, objectType: Float, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, want: want{ err: errors.ErrIndexFileNotFound, @@ -1200,7 +1188,6 @@ func Test_ngt_open(t *testing.T) { dimension: 9, objectType: Float, mu: &sync.RWMutex{}, - cmu: &sync.RWMutex{}, }, beforeFunc: func(t *testing.T, fields fields) { t.Helper() diff --git a/pkg/agent/core/ngt/handler/grpc/object.go b/pkg/agent/core/ngt/handler/grpc/object.go index b2e7a548b1..208b48bf91 100644 --- a/pkg/agent/core/ngt/handler/grpc/object.go +++ b/pkg/agent/core/ngt/handler/grpc/object.go @@ -16,6 +16,7 @@ package grpc import ( "context" "fmt" + "sync" "github.com/vdaas/vald/apis/grpc/v1/payload" "github.com/vdaas/vald/apis/grpc/v1/vald" @@ -26,7 +27,6 @@ import ( "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" "github.com/vdaas/vald/internal/observability/trace" - "github.com/vdaas/vald/internal/sync" ) func (s *server) Exists(ctx context.Context, uid *payload.Object_ID) (res *payload.Object_ID, err error) { diff --git a/versions/NGT_VERSION b/versions/NGT_VERSION index ac2cdeba01..eca07e4c1a 100644 --- a/versions/NGT_VERSION +++ b/versions/NGT_VERSION @@ -1 +1 @@ -2.1.3 +2.1.2