diff --git a/apis/docs/v1/docs.md b/apis/docs/v1/docs.md index f918850f3a8..2145f746a66 100644 --- a/apis/docs/v1/docs.md +++ b/apis/docs/v1/docs.md @@ -31,6 +31,8 @@ - [Info.IPs](#payload-v1-Info-IPs) - [Info.Index](#payload-v1-Info-Index) - [Info.Index.Count](#payload-v1-Info-Index-Count) + - [Info.Index.Detail](#payload-v1-Info-Index-Detail) + - [Info.Index.Detail.CountsEntry](#payload-v1-Info-Index-Detail-CountsEntry) - [Info.Index.UUID](#payload-v1-Info-Index-UUID) - [Info.Index.UUID.Committed](#payload-v1-Info-Index-UUID-Committed) - [Info.Index.UUID.Uncommitted](#payload-v1-Info-Index-UUID-Uncommitted) @@ -224,9 +226,10 @@ Represent the ingress filter service. Represent the index manager service. -| Method Name | Request Type | Response Type | Description | -| ----------- | -------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------- | -| IndexInfo | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the index information. | +| Method Name | Request Type | Response Type | Description | +| ----------- | -------------------------------------- | -------------------------------------------------------------- | --------------------------------------------------------------- | +| IndexInfo | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Count](#payload-v1-Info-Index-Count) | Represent the RPC to get the index information. | +| IndexDetail | [.payload.v1.Empty](#payload-v1-Empty) | [.payload.v1.Info.Index.Detail](#payload-v1-Info-Index-Detail) | Represent the RPC to get the index information for each agents. | @@ -348,6 +351,25 @@ Represent the index count message. | indexing | [bool](#bool) | | The indexing index count. | | saving | [bool](#bool) | | The saving index count. | + + +### Info.Index.Detail + +Represent the index count for each Agents message. + +| Field | Type | Label | Description | +| ------ | -------------------------------------------------------------------------- | -------- | --------------------------- | +| counts | [Info.Index.Detail.CountsEntry](#payload-v1-Info-Index-Detail-CountsEntry) | repeated | count infos for each agents | + + + +### Info.Index.Detail.CountsEntry + +| Field | Type | Label | Description | +| ----- | ------------------------------------------------ | ----- | ----------- | +| key | [string](#string) | | | +| value | [Info.Index.Count](#payload-v1-Info-Index-Count) | | | + ### Info.Index.UUID diff --git a/apis/grpc/v1/manager/index/index_manager.pb.go b/apis/grpc/v1/manager/index/index_manager.pb.go index b1d0794a490..077ef862f6f 100644 --- a/apis/grpc/v1/manager/index/index_manager.pb.go +++ b/apis/grpc/v1/manager/index/index_manager.pb.go @@ -51,31 +51,40 @@ var file_apis_proto_v1_manager_index_index_manager_proto_rawDesc = []byte{ 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x5a, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, - 0x12, 0x51, 0x0a, 0x09, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x11, 0x2e, - 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x1a, 0x1c, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, - 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x13, - 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0d, 0x12, 0x0b, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x69, - 0x6e, 0x66, 0x6f, 0x42, 0x6b, 0x0a, 0x23, 0x6f, 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, - 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x6d, 0x61, 0x6e, - 0x61, 0x67, 0x65, 0x72, 0x2e, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x10, 0x56, 0x61, 0x6c, 0x64, - 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x01, 0x5a, 0x30, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73, - 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, - 0x76, 0x31, 0x2f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xb2, 0x01, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x12, 0x51, 0x0a, 0x09, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x11, + 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x1a, 0x1c, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, + 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, + 0x13, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0d, 0x12, 0x0b, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, + 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x56, 0x0a, 0x0b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x44, 0x65, 0x74, + 0x61, 0x69, 0x6c, 0x12, 0x11, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x22, 0x15, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0f, 0x12, 0x0d, 0x2f, + 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x42, 0x6b, 0x0a, 0x23, + 0x6f, 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, + 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x2e, 0x69, 0x6e, + 0x64, 0x65, 0x78, 0x42, 0x10, 0x56, 0x61, 0x6c, 0x64, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x01, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, + 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x6d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var file_apis_proto_v1_manager_index_index_manager_proto_goTypes = []interface{}{ - (*payload.Empty)(nil), // 0: payload.v1.Empty - (*payload.Info_Index_Count)(nil), // 1: payload.v1.Info.Index.Count + (*payload.Empty)(nil), // 0: payload.v1.Empty + (*payload.Info_Index_Count)(nil), // 1: payload.v1.Info.Index.Count + (*payload.Info_Index_Detail)(nil), // 2: payload.v1.Info.Index.Detail } var file_apis_proto_v1_manager_index_index_manager_proto_depIdxs = []int32{ 0, // 0: manager.index.v1.Index.IndexInfo:input_type -> payload.v1.Empty - 1, // 1: manager.index.v1.Index.IndexInfo:output_type -> payload.v1.Info.Index.Count - 1, // [1:2] is the sub-list for method output_type - 0, // [0:1] is the sub-list for method input_type + 0, // 1: manager.index.v1.Index.IndexDetail:input_type -> payload.v1.Empty + 1, // 2: manager.index.v1.Index.IndexInfo:output_type -> payload.v1.Info.Index.Count + 2, // 3: manager.index.v1.Index.IndexDetail:output_type -> payload.v1.Info.Index.Detail + 2, // [2:4] is the sub-list for method output_type + 0, // [0:2] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/apis/grpc/v1/manager/index/index_manager_vtproto.pb.go b/apis/grpc/v1/manager/index/index_manager_vtproto.pb.go index 90f593ffa27..c62d83d9c05 100644 --- a/apis/grpc/v1/manager/index/index_manager_vtproto.pb.go +++ b/apis/grpc/v1/manager/index/index_manager_vtproto.pb.go @@ -44,6 +44,8 @@ const _ = grpc.SupportPackageIsVersion7 type IndexClient interface { // Represent the RPC to get the index information. IndexInfo(ctx context.Context, in *payload.Empty, opts ...grpc.CallOption) (*payload.Info_Index_Count, error) + // Represent the RPC to get the index information for each agents. + IndexDetail(ctx context.Context, in *payload.Empty, opts ...grpc.CallOption) (*payload.Info_Index_Detail, error) } type indexClient struct { @@ -63,12 +65,23 @@ func (c *indexClient) IndexInfo(ctx context.Context, in *payload.Empty, opts ... return out, nil } +func (c *indexClient) IndexDetail(ctx context.Context, in *payload.Empty, opts ...grpc.CallOption) (*payload.Info_Index_Detail, error) { + out := new(payload.Info_Index_Detail) + err := c.cc.Invoke(ctx, "/manager.index.v1.Index/IndexDetail", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // IndexServer is the server API for Index service. // All implementations must embed UnimplementedIndexServer // for forward compatibility type IndexServer interface { // Represent the RPC to get the index information. IndexInfo(context.Context, *payload.Empty) (*payload.Info_Index_Count, error) + // Represent the RPC to get the index information for each agents. + IndexDetail(context.Context, *payload.Empty) (*payload.Info_Index_Detail, error) mustEmbedUnimplementedIndexServer() } @@ -79,6 +92,9 @@ type UnimplementedIndexServer struct { func (UnimplementedIndexServer) IndexInfo(context.Context, *payload.Empty) (*payload.Info_Index_Count, error) { return nil, status.Errorf(codes.Unimplemented, "method IndexInfo not implemented") } +func (UnimplementedIndexServer) IndexDetail(context.Context, *payload.Empty) (*payload.Info_Index_Detail, error) { + return nil, status.Errorf(codes.Unimplemented, "method IndexDetail not implemented") +} func (UnimplementedIndexServer) mustEmbedUnimplementedIndexServer() {} // UnsafeIndexServer may be embedded to opt out of forward compatibility for this service. @@ -110,6 +126,24 @@ func _Index_IndexInfo_Handler(srv interface{}, ctx context.Context, dec func(int return interceptor(ctx, in, info, handler) } +func _Index_IndexDetail_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(payload.Empty) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(IndexServer).IndexDetail(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/manager.index.v1.Index/IndexDetail", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(IndexServer).IndexDetail(ctx, req.(*payload.Empty)) + } + return interceptor(ctx, in, info, handler) +} + // Index_ServiceDesc is the grpc.ServiceDesc for Index service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -121,6 +155,10 @@ var Index_ServiceDesc = grpc.ServiceDesc{ MethodName: "IndexInfo", Handler: _Index_IndexInfo_Handler, }, + { + MethodName: "IndexDetail", + Handler: _Index_IndexDetail_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "apis/proto/v1/manager/index/index_manager.proto", diff --git a/apis/grpc/v1/payload/payload.pb.go b/apis/grpc/v1/payload/payload.pb.go index 575f8530154..ea13813396b 100644 --- a/apis/grpc/v1/payload/payload.pb.go +++ b/apis/grpc/v1/payload/payload.pb.go @@ -4256,6 +4256,55 @@ func (x *Info_Index_Count) GetSaving() bool { return false } +// Represent the index count for each Agents message. +type Info_Index_Detail struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // count infos for each agents + Counts map[string]*Info_Index_Count `protobuf:"bytes,1,rep,name=counts,proto3" json:"counts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *Info_Index_Detail) Reset() { + *x = Info_Index_Detail{} + if protoimpl.UnsafeEnabled { + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[71] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Info_Index_Detail) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Info_Index_Detail) ProtoMessage() {} + +func (x *Info_Index_Detail) ProtoReflect() protoreflect.Message { + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[71] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Info_Index_Detail.ProtoReflect.Descriptor instead. +func (*Info_Index_Detail) Descriptor() ([]byte, []int) { + return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 1} +} + +func (x *Info_Index_Detail) GetCounts() map[string]*Info_Index_Count { + if x != nil { + return x.Counts + } + return nil +} + // Represent the UUID message. type Info_Index_UUID struct { state protoimpl.MessageState @@ -4266,7 +4315,7 @@ type Info_Index_UUID struct { func (x *Info_Index_UUID) Reset() { *x = Info_Index_UUID{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[71] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4279,7 +4328,7 @@ func (x *Info_Index_UUID) String() string { func (*Info_Index_UUID) ProtoMessage() {} func (x *Info_Index_UUID) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[71] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[72] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4292,7 +4341,7 @@ func (x *Info_Index_UUID) ProtoReflect() protoreflect.Message { // Deprecated: Use Info_Index_UUID.ProtoReflect.Descriptor instead. func (*Info_Index_UUID) Descriptor() ([]byte, []int) { - return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 1} + return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 2} } // The committed UUID. @@ -4307,7 +4356,7 @@ type Info_Index_UUID_Committed struct { func (x *Info_Index_UUID_Committed) Reset() { *x = Info_Index_UUID_Committed{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[72] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4320,7 +4369,7 @@ func (x *Info_Index_UUID_Committed) String() string { func (*Info_Index_UUID_Committed) ProtoMessage() {} func (x *Info_Index_UUID_Committed) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[72] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[74] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4333,7 +4382,7 @@ func (x *Info_Index_UUID_Committed) ProtoReflect() protoreflect.Message { // Deprecated: Use Info_Index_UUID_Committed.ProtoReflect.Descriptor instead. func (*Info_Index_UUID_Committed) Descriptor() ([]byte, []int) { - return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 1, 0} + return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 2, 0} } func (x *Info_Index_UUID_Committed) GetUuid() string { @@ -4355,7 +4404,7 @@ type Info_Index_UUID_Uncommitted struct { func (x *Info_Index_UUID_Uncommitted) Reset() { *x = Info_Index_UUID_Uncommitted{} if protoimpl.UnsafeEnabled { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[73] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4368,7 +4417,7 @@ func (x *Info_Index_UUID_Uncommitted) String() string { func (*Info_Index_UUID_Uncommitted) ProtoMessage() {} func (x *Info_Index_UUID_Uncommitted) ProtoReflect() protoreflect.Message { - mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[73] + mi := &file_apis_proto_v1_payload_payload_proto_msgTypes[75] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4381,7 +4430,7 @@ func (x *Info_Index_UUID_Uncommitted) ProtoReflect() protoreflect.Message { // Deprecated: Use Info_Index_UUID_Uncommitted.ProtoReflect.Descriptor instead. func (*Info_Index_UUID_Uncommitted) Descriptor() ([]byte, []int) { - return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 1, 1} + return file_apis_proto_v1_payload_payload_proto_rawDescGZIP(), []int{9, 0, 2, 1} } func (x *Info_Index_UUID_Uncommitted) GetUuid() string { @@ -4491,6 +4540,7 @@ var file_apis_proto_v1_payload_payload_proto_rawDesc = []byte{ 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x42, 0x09, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, +<<<<<<< HEAD 0x22, 0x6b, 0x0a, 0x14, 0x41, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, @@ -4825,6 +4875,99 @@ var file_apis_proto_v1_payload_payload_proto_rawDesc = []byte{ 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0xa2, 0x02, 0x07, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +======= + 0x22, 0x45, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x1a, 0x3a, 0x0a, 0x12, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x24, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0d, 0x42, 0x07, 0xfa, 0x42, 0x04, 0x2a, 0x02, 0x28, 0x00, 0x52, 0x08, 0x70, + 0x6f, 0x6f, 0x6c, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x66, 0x0a, 0x0a, 0x44, 0x69, 0x73, 0x63, 0x6f, + 0x76, 0x65, 0x72, 0x65, 0x72, 0x1a, 0x58, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x1b, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, + 0xfa, 0x42, 0x04, 0x72, 0x02, 0x10, 0x01, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, + 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x22, + 0x87, 0x09, 0x0a, 0x04, 0x49, 0x6e, 0x66, 0x6f, 0x1a, 0xf1, 0x02, 0x0a, 0x05, 0x49, 0x6e, 0x64, + 0x65, 0x78, 0x1a, 0x75, 0x0a, 0x05, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, + 0x74, 0x6f, 0x72, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x73, 0x74, 0x6f, + 0x72, 0x65, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x75, 0x6e, 0x63, 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x74, + 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x75, 0x6e, 0x63, 0x6f, 0x6d, 0x6d, + 0x69, 0x74, 0x74, 0x65, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x69, 0x6e, + 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x69, 0x6e, + 0x67, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x61, 0x76, 0x69, 0x6e, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x06, 0x73, 0x61, 0x76, 0x69, 0x6e, 0x67, 0x1a, 0xa4, 0x01, 0x0a, 0x06, 0x44, 0x65, + 0x74, 0x61, 0x69, 0x6c, 0x12, 0x41, 0x0a, 0x06, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x44, 0x65, 0x74, + 0x61, 0x69, 0x6c, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x06, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x73, 0x1a, 0x57, 0x0a, 0x0b, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x32, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x2e, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x1a, 0x4a, 0x0a, 0x04, 0x55, 0x55, 0x49, 0x44, 0x1a, 0x1f, 0x0a, 0x09, 0x43, 0x6f, 0x6d, 0x6d, + 0x69, 0x74, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x1a, 0x21, 0x0a, 0x0b, 0x55, 0x6e, 0x63, + 0x6f, 0x6d, 0x6d, 0x69, 0x74, 0x74, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x1a, 0xef, 0x01, 0x0a, + 0x03, 0x50, 0x6f, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x61, 0x70, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x70, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x12, 0x17, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x42, 0x07, 0xfa, + 0x42, 0x04, 0x72, 0x02, 0x78, 0x01, 0x52, 0x02, 0x69, 0x70, 0x12, 0x26, 0x0a, 0x03, 0x63, 0x70, + 0x75, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, + 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x50, 0x55, 0x52, 0x03, 0x63, + 0x70, 0x75, 0x12, 0x2f, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, + 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x52, 0x06, 0x6d, 0x65, 0x6d, + 0x6f, 0x72, 0x79, 0x12, 0x29, 0x0a, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, + 0x6e, 0x66, 0x6f, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x6e, 0x6f, 0x64, 0x65, 0x1a, 0xe8, + 0x01, 0x0a, 0x04, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61, 0x64, 0x64, 0x72, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0c, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x41, 0x64, 0x64, 0x72, + 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x61, 0x64, 0x64, + 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x78, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x41, 0x64, 0x64, 0x72, 0x12, 0x26, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, + 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x43, 0x50, 0x55, 0x52, 0x03, 0x63, 0x70, 0x75, 0x12, 0x2f, 0x0a, + 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, + 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, + 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x29, + 0x0a, 0x04, 0x50, 0x6f, 0x64, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x50, + 0x6f, 0x64, 0x73, 0x52, 0x04, 0x50, 0x6f, 0x64, 0x73, 0x1a, 0x4b, 0x0a, 0x03, 0x43, 0x50, 0x55, + 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x14, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x4e, 0x0a, 0x06, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x14, 0x0a, 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x05, 0x75, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x3a, 0x0a, 0x04, 0x50, 0x6f, 0x64, 0x73, 0x12, 0x32, + 0x0a, 0x04, 0x70, 0x6f, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x50, + 0x6f, 0x64, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x92, 0x01, 0x02, 0x08, 0x01, 0x52, 0x04, 0x70, 0x6f, + 0x64, 0x73, 0x1a, 0x3e, 0x0a, 0x05, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x05, 0x6e, + 0x6f, 0x64, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x70, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4e, 0x6f, 0x64, + 0x65, 0x42, 0x08, 0xfa, 0x42, 0x05, 0x92, 0x01, 0x02, 0x08, 0x01, 0x52, 0x05, 0x6e, 0x6f, 0x64, + 0x65, 0x73, 0x1a, 0x15, 0x0a, 0x03, 0x49, 0x50, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x42, 0x5a, 0x0a, 0x1d, 0x6f, 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, + 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x70, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x42, 0x0b, 0x56, 0x61, 0x6c, 0x64, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x50, 0x01, 0x5a, 0x2a, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x76, + 0x64, 0x61, 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x67, + 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +>>>>>>> refactor/manager-index/small-refactor } var ( @@ -4840,7 +4983,7 @@ func file_apis_proto_v1_payload_payload_proto_rawDescGZIP() []byte { } var file_apis_proto_v1_payload_payload_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_apis_proto_v1_payload_payload_proto_msgTypes = make([]protoimpl.MessageInfo, 74) +var file_apis_proto_v1_payload_payload_proto_msgTypes = make([]protoimpl.MessageInfo, 76) var file_apis_proto_v1_payload_payload_proto_goTypes = []interface{}{ (Search_AggregationAlgorithm)(0), // 0: payload.v1.Search.AggregationAlgorithm (Remove_Timestamp_Operator)(0), // 1: payload.v1.Remove.Timestamp.Operator @@ -4915,10 +5058,12 @@ var file_apis_proto_v1_payload_payload_proto_goTypes = []interface{}{ (*Info_Nodes)(nil), // 70: payload.v1.Info.Nodes (*Info_IPs)(nil), // 71: payload.v1.Info.IPs (*Info_Index_Count)(nil), // 72: payload.v1.Info.Index.Count - (*Info_Index_UUID)(nil), // 73: payload.v1.Info.Index.UUID - (*Info_Index_UUID_Committed)(nil), // 74: payload.v1.Info.Index.UUID.Committed - (*Info_Index_UUID_Uncommitted)(nil), // 75: payload.v1.Info.Index.UUID.Uncommitted - (*status.Status)(nil), // 76: google.rpc.Status + (*Info_Index_Detail)(nil), // 73: payload.v1.Info.Index.Detail + (*Info_Index_UUID)(nil), // 74: payload.v1.Info.Index.UUID + nil, // 75: payload.v1.Info.Index.Detail.CountsEntry + (*Info_Index_UUID_Committed)(nil), // 76: payload.v1.Info.Index.UUID.Committed + (*Info_Index_UUID_Uncommitted)(nil), // 77: payload.v1.Info.Index.UUID.Uncommitted + (*status.Status)(nil), // 78: google.rpc.Status } var file_apis_proto_v1_payload_payload_proto_depIdxs = []int32{ 19, // 0: payload.v1.Search.Request.config:type_name -> payload.v1.Search.Config @@ -4934,7 +5079,7 @@ var file_apis_proto_v1_payload_payload_proto_depIdxs = []int32{ 46, // 10: payload.v1.Search.Response.results:type_name -> payload.v1.Object.Distance 20, // 11: payload.v1.Search.Responses.responses:type_name -> payload.v1.Search.Response 20, // 12: payload.v1.Search.StreamResponse.response:type_name -> payload.v1.Search.Response - 76, // 13: payload.v1.Search.StreamResponse.status:type_name -> google.rpc.Status + 78, // 13: payload.v1.Search.StreamResponse.status:type_name -> google.rpc.Status 23, // 14: payload.v1.Filter.Config.targets:type_name -> payload.v1.Filter.Target 50, // 15: payload.v1.Insert.Request.vector:type_name -> payload.v1.Object.Vector 29, // 16: payload.v1.Insert.Request.config:type_name -> payload.v1.Insert.Config @@ -4968,17 +5113,17 @@ var file_apis_proto_v1_payload_payload_proto_depIdxs = []int32{ 48, // 44: payload.v1.Object.VectorRequest.id:type_name -> payload.v1.Object.ID 24, // 45: payload.v1.Object.VectorRequest.filters:type_name -> payload.v1.Filter.Config 46, // 46: payload.v1.Object.StreamDistance.distance:type_name -> payload.v1.Object.Distance - 76, // 47: payload.v1.Object.StreamDistance.status:type_name -> google.rpc.Status + 78, // 47: payload.v1.Object.StreamDistance.status:type_name -> google.rpc.Status 50, // 48: payload.v1.Object.Vectors.vectors:type_name -> payload.v1.Object.Vector 50, // 49: payload.v1.Object.StreamVector.vector:type_name -> payload.v1.Object.Vector - 76, // 50: payload.v1.Object.StreamVector.status:type_name -> google.rpc.Status + 78, // 50: payload.v1.Object.StreamVector.status:type_name -> google.rpc.Status 54, // 51: payload.v1.Object.StreamBlob.blob:type_name -> payload.v1.Object.Blob - 76, // 52: payload.v1.Object.StreamBlob.status:type_name -> google.rpc.Status + 78, // 52: payload.v1.Object.StreamBlob.status:type_name -> google.rpc.Status 56, // 53: payload.v1.Object.StreamLocation.location:type_name -> payload.v1.Object.Location - 76, // 54: payload.v1.Object.StreamLocation.status:type_name -> google.rpc.Status + 78, // 54: payload.v1.Object.StreamLocation.status:type_name -> google.rpc.Status 56, // 55: payload.v1.Object.Locations.locations:type_name -> payload.v1.Object.Location 50, // 56: payload.v1.Object.List.Response.vector:type_name -> payload.v1.Object.Vector - 76, // 57: payload.v1.Object.List.Response.status:type_name -> google.rpc.Status + 78, // 57: payload.v1.Object.List.Response.status:type_name -> google.rpc.Status 67, // 58: payload.v1.Info.Pod.cpu:type_name -> payload.v1.Info.CPU 68, // 59: payload.v1.Info.Pod.memory:type_name -> payload.v1.Info.Memory 66, // 60: payload.v1.Info.Pod.node:type_name -> payload.v1.Info.Node @@ -4987,11 +5132,13 @@ var file_apis_proto_v1_payload_payload_proto_depIdxs = []int32{ 69, // 63: payload.v1.Info.Node.Pods:type_name -> payload.v1.Info.Pods 65, // 64: payload.v1.Info.Pods.pods:type_name -> payload.v1.Info.Pod 66, // 65: payload.v1.Info.Nodes.nodes:type_name -> payload.v1.Info.Node - 66, // [66:66] is the sub-list for method output_type - 66, // [66:66] is the sub-list for method input_type - 66, // [66:66] is the sub-list for extension type_name - 66, // [66:66] is the sub-list for extension extendee - 0, // [0:66] is the sub-list for field type_name + 75, // 66: payload.v1.Info.Index.Detail.counts:type_name -> payload.v1.Info.Index.Detail.CountsEntry + 72, // 67: payload.v1.Info.Index.Detail.CountsEntry.value:type_name -> payload.v1.Info.Index.Count + 68, // [68:68] is the sub-list for method output_type + 68, // [68:68] is the sub-list for method input_type + 68, // [68:68] is the sub-list for extension type_name + 68, // [68:68] is the sub-list for extension extendee + 0, // [0:68] is the sub-list for field type_name } func init() { file_apis_proto_v1_payload_payload_proto_init() } @@ -5853,7 +6000,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { } } file_apis_proto_v1_payload_payload_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*Info_Index_UUID); i { + switch v := v.(*Info_Index_Detail); i { case 0: return &v.state case 1: @@ -5865,6 +6012,18 @@ func file_apis_proto_v1_payload_payload_proto_init() { } } file_apis_proto_v1_payload_payload_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Info_Index_UUID); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_apis_proto_v1_payload_payload_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Index_UUID_Committed); i { case 0: return &v.state @@ -5876,7 +6035,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { return nil } } - file_apis_proto_v1_payload_payload_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { + file_apis_proto_v1_payload_payload_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*Info_Index_UUID_Uncommitted); i { case 0: return &v.state @@ -5919,7 +6078,7 @@ func file_apis_proto_v1_payload_payload_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_apis_proto_v1_payload_payload_proto_rawDesc, NumEnums: 2, - NumMessages: 74, + NumMessages: 76, NumExtensions: 0, NumServices: 0, }, diff --git a/apis/grpc/v1/payload/payload_vtproto.pb.go b/apis/grpc/v1/payload/payload_vtproto.pb.go index c18557d2219..e7fc9f87965 100644 --- a/apis/grpc/v1/payload/payload_vtproto.pb.go +++ b/apis/grpc/v1/payload/payload_vtproto.pb.go @@ -1438,6 +1438,29 @@ func (m *Info_Index_Count) CloneMessageVT() proto.Message { return m.CloneVT() } +func (m *Info_Index_Detail) CloneVT() *Info_Index_Detail { + if m == nil { + return (*Info_Index_Detail)(nil) + } + r := &Info_Index_Detail{} + if rhs := m.Counts; rhs != nil { + tmpContainer := make(map[string]*Info_Index_Count, len(rhs)) + for k, v := range rhs { + tmpContainer[k] = v.CloneVT() + } + r.Counts = tmpContainer + } + if len(m.unknownFields) > 0 { + r.unknownFields = make([]byte, len(m.unknownFields)) + copy(r.unknownFields, m.unknownFields) + } + return r +} + +func (m *Info_Index_Detail) CloneMessageVT() proto.Message { + return m.CloneVT() +} + func (m *Info_Index_UUID_Committed) CloneVT() *Info_Index_UUID_Committed { if m == nil { return (*Info_Index_UUID_Committed)(nil) @@ -3591,6 +3614,42 @@ func (this *Info_Index_Count) EqualMessageVT(thatMsg proto.Message) bool { } return this.EqualVT(that) } +func (this *Info_Index_Detail) EqualVT(that *Info_Index_Detail) bool { + if this == that { + return true + } else if this == nil || that == nil { + return false + } + if len(this.Counts) != len(that.Counts) { + return false + } + for i, vx := range this.Counts { + vy, ok := that.Counts[i] + if !ok { + return false + } + if p, q := vx, vy; p != q { + if p == nil { + p = &Info_Index_Count{} + } + if q == nil { + q = &Info_Index_Count{} + } + if !p.EqualVT(q) { + return false + } + } + } + return string(this.unknownFields) == string(that.unknownFields) +} + +func (this *Info_Index_Detail) EqualMessageVT(thatMsg proto.Message) bool { + that, ok := thatMsg.(*Info_Index_Detail) + if !ok { + return false + } + return this.EqualVT(that) +} func (this *Info_Index_UUID_Committed) EqualVT(that *Info_Index_UUID_Committed) bool { if this == that { return true @@ -7088,6 +7147,61 @@ func (m *Info_Index_Count) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *Info_Index_Detail) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Info_Index_Detail) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *Info_Index_Detail) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Counts) > 0 { + for k := range m.Counts { + v := m.Counts[k] + baseI := i + size, err := v.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 + i -= len(k) + copy(dAtA[i:], k) + i = encodeVarint(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = encodeVarint(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *Info_Index_UUID_Committed) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -8924,6 +9038,29 @@ func (m *Info_Index_Count) SizeVT() (n int) { return n } +func (m *Info_Index_Detail) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Counts) > 0 { + for k, v := range m.Counts { + _ = k + _ = v + l = 0 + if v != nil { + l = v.SizeVT() + } + l += 1 + sov(uint64(l)) + mapEntrySize := 1 + len(k) + sov(uint64(len(k))) + l + n += mapEntrySize + 1 + sov(uint64(mapEntrySize)) + } + } + n += len(m.unknownFields) + return n +} + func (m *Info_Index_UUID_Committed) SizeVT() (n int) { if m == nil { return 0 @@ -15651,6 +15788,186 @@ func (m *Info_Index_Count) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *Info_Index_Detail) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Info_Index_Detail: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Info_Index_Detail: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Counts", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Counts == nil { + m.Counts = make(map[string]*Info_Index_Count) + } + var mapkey string + var mapvalue *Info_Index_Count + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLength + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLength + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLength + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLength + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &Info_Index_Count{} + if err := mapvalue.UnmarshalVT(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLength + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Counts[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *Info_Index_UUID_Committed) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/apis/proto/v1/manager/index/index_manager.proto b/apis/proto/v1/manager/index/index_manager.proto index f581c21ad9c..65a6d499209 100644 --- a/apis/proto/v1/manager/index/index_manager.proto +++ b/apis/proto/v1/manager/index/index_manager.proto @@ -32,4 +32,8 @@ service Index { rpc IndexInfo(payload.v1.Empty) returns (payload.v1.Info.Index.Count) { option (google.api.http).get = "/index/info"; } + // Represent the RPC to get the index information for each agents. + rpc IndexDetail(payload.v1.Empty) returns (payload.v1.Info.Index.Detail) { + option (google.api.http).get = "/index/detail"; + } } diff --git a/apis/proto/v1/payload/payload.proto b/apis/proto/v1/payload/payload.proto index af3638e8236..1f7860419da 100644 --- a/apis/proto/v1/payload/payload.proto +++ b/apis/proto/v1/payload/payload.proto @@ -496,6 +496,11 @@ message Info { // The saving index count. bool saving = 4; } + // Represent the index count for each Agents message. + message Detail { + // count infos for each agents + map counts = 1; + } // Represent the UUID message. message UUID { diff --git a/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json b/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json index 835e1aa5b93..51f382052f6 100644 --- a/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json +++ b/apis/swagger/v1/manager/index/apis/proto/v1/manager/index/index_manager.swagger.json @@ -7,6 +7,27 @@ "consumes": ["application/json"], "produces": ["application/json"], "paths": { + "/index/detail": { + "get": { + "summary": "Represent the RPC to get the index information for each agents.", + "operationId": "Index_IndexDetail", + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "$ref": "#/definitions/IndexDetail" + } + }, + "default": { + "description": "An unexpected error response.", + "schema": { + "$ref": "#/definitions/runtimeError" + } + } + }, + "tags": ["Index"] + } + }, "/index/info": { "get": { "summary": "Represent the RPC to get the index information.", @@ -54,6 +75,19 @@ }, "description": "Represent the index count message." }, + "IndexDetail": { + "type": "object", + "properties": { + "counts": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/IndexCount" + }, + "title": "count infos for each agents" + } + }, + "description": "Represent the index count for each Agents message." + }, "protobufAny": { "type": "object", "properties": { diff --git a/go.mod b/go.mod index d759a7989df..d93b2d90b34 100755 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ replace ( github.com/ajstarks/svgo => github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b github.com/antihax/optional => github.com/antihax/optional v1.0.0 github.com/armon/go-socks5 => github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 +<<<<<<< HEAD github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.45.6 github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 @@ -50,6 +51,15 @@ replace ( github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials v1.13.37 github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.83 +======= + github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.45.1 + github.com/aws/aws-sdk-go-v2 => github.com/aws/aws-sdk-go-v2 v1.21.0 + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream => github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 + github.com/aws/aws-sdk-go-v2/config => github.com/aws/aws-sdk-go-v2/config v1.18.38 + github.com/aws/aws-sdk-go-v2/credentials => github.com/aws/aws-sdk-go-v2/credentials v1.13.36 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds => github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 + github.com/aws/aws-sdk-go-v2/feature/s3/manager => github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.82 +>>>>>>> refactor/manager-index/small-refactor github.com/aws/aws-sdk-go-v2/internal/configsources => github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 => github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 github.com/aws/aws-sdk-go-v2/internal/ini => github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 @@ -310,8 +320,13 @@ replace ( gonum.org/v1/gonum => gonum.org/v1/gonum v0.14.0 gonum.org/v1/hdf5 => gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 gonum.org/v1/plot => gonum.org/v1/plot v0.14.0 +<<<<<<< HEAD google.golang.org/api => google.golang.org/api v0.139.0 google.golang.org/appengine => google.golang.org/appengine v1.6.8 +======= + google.golang.org/api => google.golang.org/api v0.138.0 + google.golang.org/appengine => google.golang.org/appengine v1.6.7 +>>>>>>> refactor/manager-index/small-refactor google.golang.org/genproto => google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d @@ -387,8 +402,13 @@ require ( go.uber.org/zap v1.24.0 gocloud.dev v0.0.0-00010101000000-000000000000 golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b +<<<<<<< HEAD golang.org/x/net v0.15.0 golang.org/x/oauth2 v0.12.0 +======= + golang.org/x/net v0.14.0 + golang.org/x/oauth2 v0.11.0 +>>>>>>> refactor/manager-index/small-refactor golang.org/x/sync v0.3.0 golang.org/x/sys v0.12.0 golang.org/x/text v0.13.0 @@ -491,7 +511,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.13.0 // indirect golang.org/x/exp/typeparams v0.0.0-20230307190834-24139beb5833 // indirect +<<<<<<< HEAD golang.org/x/image v0.12.0 // indirect +======= + golang.org/x/image v0.11.0 // indirect +>>>>>>> refactor/manager-index/small-refactor golang.org/x/mod v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index 04e935e5155..0dfeabfda01 100644 --- a/go.sum +++ b/go.sum @@ -175,12 +175,18 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +<<<<<<< HEAD github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.45.6 h1:Y2isQQBZsnO15dzUQo9YQRThtHgrV200XCH05BRHVJI= github.com/aws/aws-sdk-go v1.45.6/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +======= +github.com/aws/aws-sdk-go v1.45.1 h1:PXuxDZIo/Y9Bvtg2t055+dY4hRwNAEcq6bUMv9fXcjk= +github.com/aws/aws-sdk-go v1.45.1/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +>>>>>>> refactor/manager-index/small-refactor github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 h1:OPLEkmhXf6xFPiz0bLeDArZIDx1NNS4oJyG4nv3Gct0= +<<<<<<< HEAD github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13/go.mod h1:gpAbvyDGQFozTEmlTFO8XcQKHzubdq0LzRyJpG6MiXM= github.com/aws/aws-sdk-go-v2/config v1.18.39 h1:oPVyh6fuu/u4OiW4qcuQyEtk7U7uuNBmHmJSLg1AJsQ= github.com/aws/aws-sdk-go-v2/config v1.18.39/go.mod h1:+NH/ZigdPckFpgB1TRcRuWCB/Kbbvkxc/iNAKTq5RhE= @@ -190,6 +196,12 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8D github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11/go.mod h1:TEPP4tENqBGO99KwVpV9MlOX4NSrSLP8u3KRy2CDwA8= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.83 h1:wcluDLIQ0uYaxv0fCWQRimbXkPdTgWHUD21j1CzXEwc= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.83/go.mod h1:nGCBuon134gW67yAtxHKV73x+tAcY/xG4ZPNPDB1h/I= +======= +github.com/aws/aws-sdk-go-v2/config v1.18.38 h1:CByQCELMgm2tM1lAehx3XNg0R/pfeXsYzqn0Aq2chJQ= +github.com/aws/aws-sdk-go-v2/credentials v1.13.36 h1:ps0cPswZjpsOk6sLwG6fdXTzrYjCplgPEyG3OUbbdqE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 h1:uDZJF1hu0EVT/4bogChk8DyjSF6fof6uL/0Y26Ma7Fg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.82 h1:gPh2fLhr1kwH2HXFhs1kCblIgHTabqE1N9gwYPhS/fw= +>>>>>>> refactor/manager-index/small-refactor github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= @@ -207,9 +219,13 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35/go.mod h1:QG github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 h1:v0jkRigbSD6uOdwcaUQmgEwG1BkPfAPDqaeNt/29ghg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4/go.mod h1:LhTyt8J04LL+9cIt7pYJ5lbS/U98ZmXovLOR/4LUsk8= github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 h1:A42xdtStObqy7NGvzZKpnyNXvoOmm+FENobZ0/ssHWk= +<<<<<<< HEAD github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5/go.mod h1:rDGMZA7f4pbmTtPOk5v5UM2lmX6UAbRnMDJeDvnH7AM= github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 h1:2PylFCfKCEDv6PeSN09pC/VUiRd10wi1VfHG5FrW0/g= github.com/aws/aws-sdk-go-v2/service/sso v1.13.6/go.mod h1:fIAwKQKBFu90pBxx07BFOMJLpRUGu8VOzLJakeY+0K4= +======= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 h1:2PylFCfKCEDv6PeSN09pC/VUiRd10wi1VfHG5FrW0/g= +>>>>>>> refactor/manager-index/small-refactor github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1 h1:hd0SKLMdOL/Sl6Z0np1PX9LeH2gqNtBe0MhTedA8MGI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.1/go.mod h1:XO/VcyoQ8nKyKfFW/3DMsRQXsfh/052tHTWmg3xBXRg= github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 h1:CQBFElb0LS8RojMJlxRSo/HXipvTZW2S44Lt9Mk2aYQ= @@ -616,11 +632,18 @@ golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb h1:xIApU0ow1zwMa2uL1VDNeQlNV golang.org/x/exp v0.0.0-20230711153332-06a737ee72cb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/exp/shiny v0.0.0-20220827204233-334a2380cb91/go.mod h1:VjAR7z0ngyATZTELrBSkxOOHhhlnVUxDye4mcjx5h/8= golang.org/x/exp/shiny v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= +<<<<<<< HEAD golang.org/x/exp/shiny v0.0.0-20230817173708-d852ddb80c63/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9 h1:j3D9DvWRpUfIyFfDPws7LoIZ2MAI1OJHdQXtTnYtN+k= golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.12.0 h1:w13vZbU4o5rKOFFR8y7M+c4A5jXDC0uXTdHYRP8X2DQ= golang.org/x/image v0.12.0/go.mod h1:Lu90jvHG7GfemOIcldsh9A2hS01ocl6oNO7ype5mEnk= +======= +golang.org/x/exp/typeparams v0.0.0-20230817173708-d852ddb80c63 h1:XkcpbHJE31bhdecT6qfUGtB7MCIKA8Vb9uGOyX/T364= +golang.org/x/exp/typeparams v0.0.0-20230817173708-d852ddb80c63/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= +golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= +>>>>>>> refactor/manager-index/small-refactor golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20230906132913-2077a3224571/go.mod h1:wEyOn6VvNW7tcf+bW/wBz1sehi2s2BZ4TimyR7qZen4= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= @@ -651,10 +674,17 @@ gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENE gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I= gonum.org/v1/plot v0.14.0 h1:+LBDVFYwFe4LHhdP8coW6296MBEY4nQ+Y4vuUpJopcE= gonum.org/v1/plot v0.14.0/go.mod h1:MLdR9424SJed+5VqC6MsouEpig9pZX2VZ57H9ko2bXU= +<<<<<<< HEAD google.golang.org/api v0.139.0 h1:A1TrCPgMmOiYu0AiNkvQIpIx+D8blHTDcJ5EogkP7LI= google.golang.org/api v0.139.0/go.mod h1:CVagp6Eekz9CjGZ718Z+sloknzkDJE7Vc1Ckj9+viBk= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +======= +google.golang.org/api v0.138.0 h1:K/tVp05MxNVbHShRw9m7e9VJGdagNeTdMzqPH7AUqr0= +google.golang.org/api v0.138.0/go.mod h1:4xyob8CxC+0GChNBvEUAk8VBKNvYOTWM9T3v3UfRxuY= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +>>>>>>> refactor/manager-index/small-refactor google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= diff --git a/internal/observability/trace/status.go b/internal/observability/trace/status.go index c1b08de2e43..bd03c296dc7 100644 --- a/internal/observability/trace/status.go +++ b/internal/observability/trace/status.go @@ -19,7 +19,7 @@ package trace import ( "github.com/vdaas/vald/internal/net/grpc/codes" - "go.opentelemetry.io/otel/attribute" + "github.com/vdaas/vald/internal/observability/attribute" ocodes "go.opentelemetry.io/otel/codes" semconv "go.opentelemetry.io/otel/semconv/v1.12.0" ) diff --git a/pkg/agent/core/ngt/handler/grpc/insert.go b/pkg/agent/core/ngt/handler/grpc/insert.go index 172e256b60d..a78f98db791 100644 --- a/pkg/agent/core/ngt/handler/grpc/insert.go +++ b/pkg/agent/core/ngt/handler/grpc/insert.go @@ -26,9 +26,9 @@ import ( "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/strings" - "go.opentelemetry.io/otel/attribute" ) // Insert inserts a vector to the NGT. diff --git a/pkg/agent/core/ngt/handler/grpc/linear_search.go b/pkg/agent/core/ngt/handler/grpc/linear_search.go index 9ed55f04bb3..0beb2512968 100644 --- a/pkg/agent/core/ngt/handler/grpc/linear_search.go +++ b/pkg/agent/core/ngt/handler/grpc/linear_search.go @@ -26,11 +26,11 @@ import ( "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync" - "go.opentelemetry.io/otel/attribute" ) func (s *server) LinearSearch(ctx context.Context, req *payload.Search_Request) (res *payload.Search_Response, err error) { diff --git a/pkg/agent/core/ngt/handler/grpc/remove.go b/pkg/agent/core/ngt/handler/grpc/remove.go index 50d64792358..16a06a1e3f5 100644 --- a/pkg/agent/core/ngt/handler/grpc/remove.go +++ b/pkg/agent/core/ngt/handler/grpc/remove.go @@ -26,10 +26,10 @@ import ( "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync" - "go.opentelemetry.io/otel/attribute" ) func (s *server) Remove(ctx context.Context, req *payload.Remove_Request) (res *payload.Object_Location, err error) { diff --git a/pkg/agent/core/ngt/handler/grpc/search.go b/pkg/agent/core/ngt/handler/grpc/search.go index b2149ab4fd8..fcf749a29ac 100644 --- a/pkg/agent/core/ngt/handler/grpc/search.go +++ b/pkg/agent/core/ngt/handler/grpc/search.go @@ -26,11 +26,11 @@ import ( "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync" - "go.opentelemetry.io/otel/attribute" ) func (s *server) Search(ctx context.Context, req *payload.Search_Request) (res *payload.Search_Response, err error) { diff --git a/pkg/agent/core/ngt/handler/grpc/update.go b/pkg/agent/core/ngt/handler/grpc/update.go index bcefc777014..0174195ad09 100644 --- a/pkg/agent/core/ngt/handler/grpc/update.go +++ b/pkg/agent/core/ngt/handler/grpc/update.go @@ -26,9 +26,9 @@ import ( "github.com/vdaas/vald/internal/net/grpc/codes" "github.com/vdaas/vald/internal/net/grpc/errdetails" "github.com/vdaas/vald/internal/net/grpc/status" + "github.com/vdaas/vald/internal/observability/attribute" "github.com/vdaas/vald/internal/observability/trace" "github.com/vdaas/vald/internal/strings" - "go.opentelemetry.io/otel/attribute" ) func (s *server) Update(ctx context.Context, req *payload.Update_Request) (res *payload.Object_Location, err error) { diff --git a/pkg/agent/core/ngt/service/ngt_test.go b/pkg/agent/core/ngt/service/ngt_test.go index 1153d4308e7..01203d394e5 100644 --- a/pkg/agent/core/ngt/service/ngt_test.go +++ b/pkg/agent/core/ngt/service/ngt_test.go @@ -37,6 +37,7 @@ import ( "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/file" "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/safety" "github.com/vdaas/vald/internal/strings" "github.com/vdaas/vald/internal/sync" @@ -47,7 +48,6 @@ import ( "github.com/vdaas/vald/pkg/agent/core/ngt/service/kvs" "github.com/vdaas/vald/pkg/agent/core/ngt/service/vqueue" "github.com/vdaas/vald/pkg/agent/internal/metadata" - "google.golang.org/grpc" ) var defaultConfig = config.NGT{ @@ -1113,8 +1113,8 @@ func Test_ngt_E2E(t *testing.T) { type args struct { requests []*payload.Upsert_MultiRequest - addr string - dialOpts []grpc.DialOption + addr string + client grpc.Client } type want struct { err error @@ -1169,10 +1169,8 @@ func Test_ngt_E2E(t *testing.T) { createRandomData(500000, new(createRandomDataConfig)), 50, ), - addr: "127.0.0.1:8080", - dialOpts: []grpc.DialOption{ - grpc.WithInsecure(), - }, + addr: "127.0.0.1:8080", + client: grpc.New(grpc.WithInsecure(true)), }, }, } @@ -1194,20 +1192,15 @@ func Test_ngt_E2E(t *testing.T) { if test.checkFunc == nil { checkFunc = defaultCheckFunc } - conn, err := grpc.DialContext(ctx, test.args.addr, test.args.dialOpts...) - if err := checkFunc(test.want, err); err != nil { - t.Fatal(err) - } - defer func() { - if err := conn.Close(); err != nil { - t.Error(err) - } - }() - client := vald.NewValdClient(conn) + + defer test.args.client.Close(ctx) for i := 0; i < 2; i++ { for _, req := range test.args.requests { - _, err := client.MultiUpsert(ctx, req) + _, err := test.args.client.Do(ctx, test.args.addr, + func(ctx context.Context, conn *grpc.ClientConn, opts ...grpc.CallOption) (any, error) { + return vald.NewValdClient(conn).MultiInsert(ctx, req) + }) if err != nil { t.Error(err) } diff --git a/pkg/manager/index/handler/grpc/handler.go b/pkg/manager/index/handler/grpc/handler.go index 240a8c20185..06415a7892b 100644 --- a/pkg/manager/index/handler/grpc/handler.go +++ b/pkg/manager/index/handler/grpc/handler.go @@ -53,3 +53,13 @@ func (s *server) IndexInfo(ctx context.Context, _ *payload.Empty) (res *payload. Indexing: s.indexer.IsIndexing(), }, nil } + +func (s *server) IndexDetail(ctx context.Context, _ *payload.Empty) (res *payload.Info_Index_Detail, err error) { + ctx, span := trace.StartSpan(ctx, "vald/manager-index.IndexDetail") + defer func() { + if span != nil { + span.End() + } + }() + return s.indexer.LoadIndexDetail(), nil +} diff --git a/pkg/manager/index/handler/grpc/handler_test.go b/pkg/manager/index/handler/grpc/handler_test.go index 44d3209d1e4..ba928ed4327 100644 --- a/pkg/manager/index/handler/grpc/handler_test.go +++ b/pkg/manager/index/handler/grpc/handler_test.go @@ -214,3 +214,113 @@ package grpc // }) // } // } +// +// func Test_server_IndexDetail(t *testing.T) { +// type args struct { +// ctx context.Context +// in1 *payload.Empty +// } +// type fields struct { +// indexer service.Indexer +// UnimplementedIndexServer index.UnimplementedIndexServer +// } +// type want struct { +// wantRes *payload.Info_Index_Detail +// err error +// } +// type test struct { +// name string +// args args +// fields fields +// want want +// checkFunc func(want, *payload.Info_Index_Detail, error) error +// beforeFunc func(*testing.T, args) +// afterFunc func(*testing.T, args) +// } +// defaultCheckFunc := func(w want, gotRes *payload.Info_Index_Detail, err error) error { +// if !errors.Is(err, w.err) { +// return errors.Errorf("got_error: \"%#v\",\n\t\t\t\twant: \"%#v\"", err, w.err) +// } +// if !reflect.DeepEqual(gotRes, w.wantRes) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotRes, w.wantRes) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// args: args { +// ctx:nil, +// in1:nil, +// }, +// fields: fields { +// indexer:nil, +// UnimplementedIndexServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// args: args { +// ctx:nil, +// in1:nil, +// }, +// fields: fields { +// indexer:nil, +// UnimplementedIndexServer:nil, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T, args args) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt, test.args) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt, test.args) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// s := &server{ +// indexer: test.fields.indexer, +// UnimplementedIndexServer: test.fields.UnimplementedIndexServer, +// } +// +// gotRes, err := s.IndexDetail(test.args.ctx, test.args.in1) +// if err := checkFunc(test.want, gotRes, err); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/manager/index/service/indexer.go b/pkg/manager/index/service/indexer.go index 6eb28eaad90..8ac4dc7e772 100644 --- a/pkg/manager/index/service/indexer.go +++ b/pkg/manager/index/service/indexer.go @@ -43,6 +43,8 @@ type Indexer interface { NumberOfUUIDs() uint32 NumberOfUncommittedUUIDs() uint32 IsIndexing() bool + IsSaving() bool + LoadIndexDetail() *payload.Info_Index_Detail } type index struct { @@ -51,18 +53,21 @@ type index struct { creationPoolSize uint32 indexDuration time.Duration indexDurationLimit time.Duration + saveIndexDuration time.Duration saveIndexDurationLimit time.Duration - saveIndexWaitDuration time.Duration - saveIndexTargetAddrCh chan string - schMap sync.Map[string, any] - concurrency int + shouldSaveList sync.Map[string, struct{}] + createIndexConcurrency int + saveIndexConcurrency int indexInfos sync.Map[string, *payload.Info_Index_Count] - indexing atomic.Value // bool + indexing atomic.Bool + saving atomic.Bool minUncommitted uint32 uuidsCount uint32 uncommittedUUIDsCount uint32 } +var empty = struct{}{} + func New(opts ...Option) (idx Indexer, err error) { i := new(index) for _, opt := range append(defaultOptions, opts...) { @@ -71,7 +76,8 @@ func New(opts ...Option) (idx Indexer, err error) { } } i.indexing.Store(false) - if i.indexDuration+i.indexDurationLimit+i.saveIndexDurationLimit == 0 { + i.saving.Store(false) + if i.indexDuration+i.indexDurationLimit+i.saveIndexDurationLimit <= 0 { return nil, errors.ErrInvalidConfig } return i, nil @@ -87,8 +93,6 @@ func (idx *index) Start(ctx context.Context) (<-chan error, error) { return nil, err } ech := make(chan error, 100) - sech := make(chan error, 10) - idx.saveIndexTargetAddrCh = make(chan string, len(idx.client.GetAddrs(ctx))*2) idx.eg.Go(safety.RecoverFunc(func() (err error) { defer close(ech) if idx.indexDuration <= 0 { @@ -97,56 +101,101 @@ func (idx *index) Start(ctx context.Context) (<-chan error, error) { if idx.indexDurationLimit <= 0 { idx.indexDurationLimit = math.MaxInt64 } + if idx.saveIndexDuration <= 0 { + idx.saveIndexDuration = math.MaxInt64 + } if idx.saveIndexDurationLimit <= 0 { idx.saveIndexDurationLimit = math.MaxInt64 } it := time.NewTicker(idx.indexDuration) itl := time.NewTicker(idx.indexDurationLimit) + st := time.NewTicker(idx.saveIndexDuration) stl := time.NewTicker(idx.saveIndexDurationLimit) defer it.Stop() defer itl.Stop() + defer st.Stop() defer stl.Stop() finalize := func() (err error) { err = ctx.Err() - if err != nil && err != context.Canceled { + if err != nil && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { return err } return nil } + var mu sync.Mutex for { select { case <-ctx.Done(): return finalize() case err = <-dech: - ech <- err - case err = <-sech: - ech <- err - case <-it.C: - err = idx.execute(grpc.WithGRPCMethod(ctx, "core.v1.Agent/CreateIndex"), true, false) - if err != nil { - ech <- err - log.Error("an error occurred during indexing", err) - err = nil + case <-it.C: // index duration ticker + // execute CreateIndex. This execution ignores low index agent. + err = idx.createIndex(grpc.WithGRPCMethod(ctx, "core.v1.Agent/CreateIndex"), true) + if err != nil && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + err = errors.Wrap(err, "an error occurred during create indexing") } it.Reset(idx.indexDuration) - case <-itl.C: - err = idx.execute(grpc.WithGRPCMethod(ctx, "core.v1.Agent/CreateIndex"), false, false) - if err != nil { - ech <- err - log.Error("an error occurred during indexing", err) - err = nil + case <-itl.C: // index duration limit ticker + // execute CreateIndex. This execution always executes CreateIndex regardless of the state of the uncommitted index. + err = idx.createIndex(grpc.WithGRPCMethod(ctx, "core.v1.Agent/CreateIndex"), false) + if err != nil && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + err = errors.Wrap(err, "an error occurred during force create indexing") } itl.Reset(idx.indexDurationLimit) - case <-stl.C: - err = idx.execute(grpc.WithGRPCMethod(ctx, "core.v1.Agent/CreateAndSaveIndex"), false, true) - if err != nil { - ech <- err - log.Error("an error occurred during indexing and saving", err) - err = nil - } - stl.Reset(idx.saveIndexDurationLimit) + case <-st.C: // save index duration ticker + // execute SaveIndex in concurrent. + idx.eg.Go(safety.RecoverFunc(func() (err error) { + if !mu.TryLock() { + return + } + defer mu.Unlock() + defer st.Reset(idx.saveIndexDuration) + err = idx.saveIndex(grpc.WithGRPCMethod(ctx, "core.v1.Agent/SaveIndex"), false) + if err != nil && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + err = errors.Wrap(err, "an error occurred during save indexing") + log.Error(err) + select { + case <-ctx.Done(): + return nil + case ech <- err: + } + } + return nil + })) + case <-stl.C: // save index duration limit ticker + // execute SaveIndex in concurrent. + idx.eg.Go(safety.RecoverFunc(func() (err error) { + if !mu.TryLock() { + return + } + defer mu.Unlock() + defer stl.Reset(idx.saveIndexDurationLimit) + err = idx.saveIndex(grpc.WithGRPCMethod(ctx, "core.v1.Agent/SaveIndex"), true) + if err != nil && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { + err = errors.Wrap(err, "an error occurred during force save indexing") + log.Error(err) + select { + case <-ctx.Done(): + return nil + case ech <- err: + } + } + return nil + })) } - if err != nil { + if err != nil && + !errors.Is(err, context.Canceled) && + !errors.Is(err, context.DeadlineExceeded) { log.Error(err) select { case <-ctx.Done(): @@ -156,35 +205,10 @@ func (idx *index) Start(ctx context.Context) (<-chan error, error) { } } })) - idx.eg.Go(safety.RecoverFunc(func() (err error) { - defer close(sech) - for { - select { - case <-ctx.Done(): - return - case addr := <-idx.saveIndexTargetAddrCh: - idx.schMap.Delete(addr) - _, err := idx.client.GetClient(). - Do(grpc.WithGRPCMethod(ctx, "core.v1.Agent/SaveIndex"), addr, func(ctx context.Context, conn *grpc.ClientConn, copts ...grpc.CallOption) (interface{}, error) { - return agent.NewAgentClient(conn).SaveIndex(ctx, &payload.Empty{}, copts...) - }) - if err != nil { - log.Warnf("an error occurred while calling SaveIndex of %s: %s", addr, err) - select { - case <-ctx.Done(): - return nil - case sech <- err: - } - } - } - - idx.waitForNextSaving(ctx) - } - })) return ech, nil } -func (idx *index) execute(ctx context.Context, enableLowIndexSkip, immediateSaving bool) (err error) { +func (idx *index) createIndex(ctx context.Context, enableLowIndexSkip bool) (err error) { ctx, span := trace.StartSpan(ctx, "vald/manager-index/service/Indexer.execute") defer func() { if span != nil { @@ -192,79 +216,75 @@ func (idx *index) execute(ctx context.Context, enableLowIndexSkip, immediateSavi } }() - if idx.indexing.Load().(bool) { + if idx.indexing.Load() { return nil } idx.indexing.Store(true) defer idx.indexing.Store(false) - addrs := idx.client.GetAddrs(ctx) - err = idx.client.GetClient().OrderedRangeConcurrent(ctx, addrs, - idx.concurrency, + return errors.Join(idx.client.GetClient().OrderedRangeConcurrent(ctx, idx.client.GetAddrs(ctx), + idx.createIndexConcurrency, func(ctx context.Context, addr string, conn *grpc.ClientConn, copts ...grpc.CallOption, ) (err error) { - select { - case <-ctx.Done(): - return nil - default: - } info, ok := idx.indexInfos.Load(addr) if ok && (info.GetUncommitted() == 0 || (enableLowIndexSkip && info.GetUncommitted() < idx.minUncommitted)) { return nil } - ac := agent.NewAgentClient(conn) - req := &payload.Control_CreateIndexRequest{ + _, err = agent.NewAgentClient(conn).CreateIndex(ctx, &payload.Control_CreateIndexRequest{ PoolSize: idx.creationPoolSize, - } - if !immediateSaving { - _, err = ac.CreateIndex(ctx, req, copts...) - if err != nil { - st, ok := status.FromError(err) - if ok && st != nil && st.Code() == codes.FailedPrecondition { - log.Debugf("CreateIndex of %s skipped, message: %s, err: %v", addr, st.Message(), errors.Join(st.Err(), err)) - return nil - } - log.Warnf("an error occurred while calling CreateIndex of %s: %s", addr, err) - return err - } - _, ok := idx.schMap.Load(addr) - if !ok { - select { - case <-ctx.Done(): - case idx.saveIndexTargetAddrCh <- addr: - idx.schMap.Store(addr, struct{}{}) - } - } - return nil - } - _, err = ac.CreateAndSaveIndex(ctx, req, copts...) + }, copts...) if err != nil { st, ok := status.FromError(err) if ok && st != nil && st.Code() == codes.FailedPrecondition { log.Debugf("CreateIndex of %s skipped, message: %s, err: %v", addr, st.Message(), errors.Join(st.Err(), err)) return nil } - log.Warnf("an error occurred while calling CreateAndSaveIndex of %s: %s", addr, err) + log.Warnf("an error occurred while calling CreateIndex of %s: %s", addr, err) return err } - idx.waitForNextSaving(ctx) + _, ok = idx.shouldSaveList.LoadOrStore(addr, empty) + if ok { + log.Debugf("addr %s already queued for saveIndex", addr) + return nil + } return nil - }) - if err != nil { - return err - } - return idx.loadInfos(ctx) + }), idx.loadInfos(ctx)) } -func (idx *index) waitForNextSaving(ctx context.Context) { - if idx.saveIndexWaitDuration > 0 { - timer := time.NewTimer(idx.saveIndexWaitDuration) - select { - case <-ctx.Done(): - case <-timer.C: +func (idx *index) saveIndex(ctx context.Context, force bool) (err error) { + ctx, span := trace.StartSpan(ctx, "vald/manager-index/service/Indexer.saveIndex") + defer func() { + if span != nil { + span.End() } - timer.Stop() + }() + + if idx.saving.Load() { + return nil } + idx.saving.Store(true) + defer idx.saving.Store(false) + return idx.client.GetClient().OrderedRangeConcurrent(ctx, idx.client.GetAddrs(ctx), + idx.saveIndexConcurrency, + func(ctx context.Context, + addr string, conn *grpc.ClientConn, copts ...grpc.CallOption, + ) (err error) { + _, ok := idx.shouldSaveList.LoadAndDelete(addr) + if !ok && !force { + return nil + } + _, err = agent.NewAgentClient(conn).SaveIndex(ctx, new(payload.Empty), copts...) + if err != nil { + st, ok := status.FromError(err) + if ok && st != nil && st.Code() == codes.FailedPrecondition { + log.Debugf("CreateIndex of %s skipped, message: %s, err: %v", addr, st.Message(), errors.Join(st.Err(), err)) + return nil + } + log.Warnf("an error occurred while calling CreateIndex of %s: %s", addr, err) + return err + } + return nil + }) } func (idx *index) loadInfos(ctx context.Context) (err error) { @@ -318,7 +338,11 @@ func (idx *index) loadInfos(ctx context.Context) (err error) { } func (idx *index) IsIndexing() bool { - return idx.indexing.Load().(bool) + return idx.indexing.Load() +} + +func (idx *index) IsSaving() bool { + return idx.saving.Load() } func (idx *index) NumberOfUUIDs() uint32 { @@ -328,3 +352,12 @@ func (idx *index) NumberOfUUIDs() uint32 { func (idx *index) NumberOfUncommittedUUIDs() uint32 { return atomic.LoadUint32(&idx.uncommittedUUIDsCount) } + +func (idx *index) LoadIndexDetail() (detail *payload.Info_Index_Detail) { + detail = new(payload.Info_Index_Detail) + idx.indexInfos.Range(func(addr string, info *payload.Info_Index_Count) bool { + detail.Counts[addr] = info + return true + }) + return detail +} diff --git a/pkg/manager/index/service/indexer_test.go b/pkg/manager/index/service/indexer_test.go index 4a8942d6455..2c9fd644cfd 100644 --- a/pkg/manager/index/service/indexer_test.go +++ b/pkg/manager/index/service/indexer_test.go @@ -702,3 +702,148 @@ package service // }) // } // } +// +// func Test_index_LoadIndexDetail(t *testing.T) { +// type fields struct { +// client discoverer.Client +// eg errgroup.Group +// creationPoolSize uint32 +// indexDuration time.Duration +// indexDurationLimit time.Duration +// saveIndexDurationLimit time.Duration +// saveIndexWaitDuration time.Duration +// saveIndexTargetAddrCh chan string +// schMap sync.Map[string, any] +// concurrency int +// indexInfos sync.Map[string, *payload.Info_Index_Count] +// indexing atomic.Value +// minUncommitted uint32 +// uuidsCount uint32 +// uncommittedUUIDsCount uint32 +// } +// type want struct { +// wantDetail *payload.Info_Index_Detail +// } +// type test struct { +// name string +// fields fields +// want want +// checkFunc func(want, *payload.Info_Index_Detail) error +// beforeFunc func(*testing.T) +// afterFunc func(*testing.T) +// } +// defaultCheckFunc := func(w want, gotDetail *payload.Info_Index_Detail) error { +// if !reflect.DeepEqual(gotDetail, w.wantDetail) { +// return errors.Errorf("got: \"%#v\",\n\t\t\t\twant: \"%#v\"", gotDetail, w.wantDetail) +// } +// return nil +// } +// tests := []test{ +// // TODO test cases +// /* +// { +// name: "test_case_1", +// fields: fields { +// client:nil, +// eg:nil, +// creationPoolSize:0, +// indexDuration:nil, +// indexDurationLimit:nil, +// saveIndexDurationLimit:nil, +// saveIndexWaitDuration:nil, +// saveIndexTargetAddrCh:nil, +// schMap:nil, +// concurrency:0, +// indexInfos:nil, +// indexing:nil, +// minUncommitted:0, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// }, +// */ +// +// // TODO test cases +// /* +// func() test { +// return test { +// name: "test_case_2", +// fields: fields { +// client:nil, +// eg:nil, +// creationPoolSize:0, +// indexDuration:nil, +// indexDurationLimit:nil, +// saveIndexDurationLimit:nil, +// saveIndexWaitDuration:nil, +// saveIndexTargetAddrCh:nil, +// schMap:nil, +// concurrency:0, +// indexInfos:nil, +// indexing:nil, +// minUncommitted:0, +// uuidsCount:0, +// uncommittedUUIDsCount:0, +// }, +// want: want{}, +// checkFunc: defaultCheckFunc, +// beforeFunc: func(t *testing.T,) { +// t.Helper() +// }, +// afterFunc: func(t *testing.T,) { +// t.Helper() +// }, +// } +// }(), +// */ +// } +// +// for _, tc := range tests { +// test := tc +// t.Run(test.name, func(tt *testing.T) { +// tt.Parallel() +// defer goleak.VerifyNone(tt, goleak.IgnoreCurrent()) +// if test.beforeFunc != nil { +// test.beforeFunc(tt) +// } +// if test.afterFunc != nil { +// defer test.afterFunc(tt) +// } +// checkFunc := test.checkFunc +// if test.checkFunc == nil { +// checkFunc = defaultCheckFunc +// } +// idx := &index{ +// client: test.fields.client, +// eg: test.fields.eg, +// creationPoolSize: test.fields.creationPoolSize, +// indexDuration: test.fields.indexDuration, +// indexDurationLimit: test.fields.indexDurationLimit, +// saveIndexDurationLimit: test.fields.saveIndexDurationLimit, +// saveIndexWaitDuration: test.fields.saveIndexWaitDuration, +// saveIndexTargetAddrCh: test.fields.saveIndexTargetAddrCh, +// schMap: test.fields.schMap, +// concurrency: test.fields.concurrency, +// indexInfos: test.fields.indexInfos, +// indexing: test.fields.indexing, +// minUncommitted: test.fields.minUncommitted, +// uuidsCount: test.fields.uuidsCount, +// uncommittedUUIDsCount: test.fields.uncommittedUUIDsCount, +// } +// +// gotDetail := idx.LoadIndexDetail() +// if err := checkFunc(test.want, gotDetail); err != nil { +// tt.Errorf("error = %v", err) +// } +// +// }) +// } +// } diff --git a/pkg/manager/index/service/option.go b/pkg/manager/index/service/option.go index d3198a93fd5..85307038dde 100644 --- a/pkg/manager/index/service/option.go +++ b/pkg/manager/index/service/option.go @@ -31,7 +31,6 @@ var defaultOptions = []Option{ WithIndexingDuration("1m"), WithIndexingDurationLimit("30m"), WithSaveIndexDurationLimit("3h"), - WithSaveIndexWaitDuration("10m"), WithMinUncommitted(100), WithCreationPoolSize(10000), } @@ -39,27 +38,22 @@ var defaultOptions = []Option{ func WithIndexingConcurrency(c int) Option { return func(idx *index) error { if c != 0 { - idx.concurrency = c + idx.createIndexConcurrency = c } return nil } } -func WithIndexingDuration(dur string) Option { +func WithSaveConcurrency(c int) Option { return func(idx *index) error { - if dur == "" { - return nil - } - d, err := timeutil.Parse(dur) - if err != nil { - return err + if c != 0 { + idx.saveIndexConcurrency = c } - idx.indexDuration = d return nil } } -func WithIndexingDurationLimit(dur string) Option { +func WithIndexingDuration(dur string) Option { return func(idx *index) error { if dur == "" { return nil @@ -68,12 +62,12 @@ func WithIndexingDurationLimit(dur string) Option { if err != nil { return err } - idx.indexDurationLimit = d + idx.indexDuration = d return nil } } -func WithSaveIndexDurationLimit(dur string) Option { +func WithIndexingDurationLimit(dur string) Option { return func(idx *index) error { if dur == "" { return nil @@ -82,12 +76,12 @@ func WithSaveIndexDurationLimit(dur string) Option { if err != nil { return err } - idx.saveIndexDurationLimit = d + idx.indexDurationLimit = d return nil } } -func WithSaveIndexWaitDuration(dur string) Option { +func WithSaveIndexDurationLimit(dur string) Option { return func(idx *index) error { if dur == "" { return nil @@ -96,7 +90,7 @@ func WithSaveIndexWaitDuration(dur string) Option { if err != nil { return err } - idx.saveIndexWaitDuration = d + idx.saveIndexDurationLimit = d return nil } } diff --git a/pkg/manager/index/usecase/indexer.go b/pkg/manager/index/usecase/indexer.go index 11cc68c85f6..cf361892f6a 100644 --- a/pkg/manager/index/usecase/indexer.go +++ b/pkg/manager/index/usecase/indexer.go @@ -98,7 +98,6 @@ func New(cfg *config.Data) (r runner.Runner, err error) { service.WithIndexingDuration(cfg.Indexer.AutoIndexCheckDuration), service.WithIndexingDurationLimit(cfg.Indexer.AutoIndexDurationLimit), service.WithSaveIndexDurationLimit(cfg.Indexer.AutoSaveIndexDurationLimit), - service.WithSaveIndexWaitDuration(cfg.Indexer.AutoSaveIndexWaitDuration), service.WithCreationPoolSize(cfg.Indexer.CreationPoolSize), service.WithMinUncommitted(cfg.Indexer.AutoIndexLength), )