refactor proto

Signed-off-by: kpango <[email protected]>
vdaas · Sep 10, 2023 · c0e98b6 · c0e98b6
1 parent 5b9dd88
commit c0e98b6
Show file tree

Hide file tree

Showing 17 changed files with 812 additions and 466 deletions.
diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk
@@ -31,12 +31,16 @@ endef
 define proto-code-gen
 	protoc \
 		$(PROTO_PATHS:%=-I %) \
-                --go_out=$(GOPATH)/src --plugin protoc-gen-go="$(GOBIN)/protoc-gen-go" \
-                --go-vtproto_out=$(GOPATH)/src --plugin protoc-gen-go-vtproto="$(GOBIN)/protoc-gen-go-vtproto" \
-                --go-vtproto_opt=features=grpc+marshal+unmarshal+size+equal+clone \
+                --go_out=$(GOPATH)/src \
+		--plugin protoc-gen-go="$(GOBIN)/protoc-gen-go" \
+                --go-vtproto_out=$(GOPATH)/src \
+		--plugin protoc-gen-go-vtproto="$(GOBIN)/protoc-gen-go-vtproto" \
+                --go-vtproto_opt=features=grpc+marshal+unmarshal+size+equal+clone+pool \
+                --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Object.Vector \
 		$1
 endef
-                # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Object.Vector \
+		# --go-grpc_out=$(GOPATH)/src \
+		# --plugin protoc-gen-go-grpc="${GOBIN}/protoc-gen-go-grpc" \
                 # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Insert.MultiRequest \
                 # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Insert.Request \
                 # --go-vtproto_opt=pool=$(ROOTDIR)/apis/proto/v1/payload.Object.Vector \

diff --git a/apis/grpc/v1/filter/egress/egress_filter_vtproto.pb.go b/apis/grpc/v1/filter/egress/egress_filter_vtproto.pb.go
@@ -66,7 +66,7 @@ func (c *filterClient) FilterDistance(ctx context.Context, in *payload.Object_Di
 }
 
 func (c *filterClient) FilterVector(ctx context.Context, in *payload.Object_Vector, opts ...grpc.CallOption) (*payload.Object_Vector, error) {
-	out := new(payload.Object_Vector)
+	out := payload.Object_VectorFromVTPool()
 	err := c.cc.Invoke(ctx, "/filter.egress.v1.Filter/FilterVector", in, out, opts...)
 	if err != nil {
 		return nil, err
@@ -127,7 +127,7 @@ func _Filter_FilterDistance_Handler(srv interface{}, ctx context.Context, dec fu
 }
 
 func _Filter_FilterVector_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(payload.Object_Vector)
+	in := payload.Object_VectorFromVTPool()
 	if err := dec(in); err != nil {
 		return nil, err
 	}

diff --git a/apis/grpc/v1/filter/ingress/ingress_filter_vtproto.pb.go b/apis/grpc/v1/filter/ingress/ingress_filter_vtproto.pb.go
@@ -57,7 +57,7 @@ func NewFilterClient(cc grpc.ClientConnInterface) FilterClient {
 }
 
 func (c *filterClient) GenVector(ctx context.Context, in *payload.Object_Blob, opts ...grpc.CallOption) (*payload.Object_Vector, error) {
-	out := new(payload.Object_Vector)
+	out := payload.Object_VectorFromVTPool()
 	err := c.cc.Invoke(ctx, "/filter.ingress.v1.Filter/GenVector", in, out, opts...)
 	if err != nil {
 		return nil, err
@@ -66,7 +66,7 @@ func (c *filterClient) GenVector(ctx context.Context, in *payload.Object_Blob, o
 }
 
 func (c *filterClient) FilterVector(ctx context.Context, in *payload.Object_Vector, opts ...grpc.CallOption) (*payload.Object_Vector, error) {
-	out := new(payload.Object_Vector)
+	out := payload.Object_VectorFromVTPool()
 	err := c.cc.Invoke(ctx, "/filter.ingress.v1.Filter/FilterVector", in, out, opts...)
 	if err != nil {
 		return nil, err
@@ -127,7 +127,7 @@ func _Filter_GenVector_Handler(srv interface{}, ctx context.Context, dec func(in
 }
 
 func _Filter_FilterVector_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(payload.Object_Vector)
+	in := payload.Object_VectorFromVTPool()
 	if err := dec(in); err != nil {
 		return nil, err
 	}

diff --git a/apis/grpc/v1/payload/payload.pb.go b/apis/grpc/v1/payload/payload.pb.go
diff --git a/apis/grpc/v1/payload/payload_vtproto.pb.go b/apis/grpc/v1/payload/payload_vtproto.pb.go
@@ -23,6 +23,7 @@ import (
 	bits "math/bits"
 
 	io "github.com/vdaas/vald/internal/io"
+	sync "github.com/vdaas/vald/internal/sync"
 	status "google.golang.org/genproto/googleapis/rpc/status"
 	proto "google.golang.org/protobuf/proto"
 	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
@@ -7719,6 +7720,27 @@ func encodeVarint(dAtA []byte, offset int, v uint64) int {
 	dAtA[offset] = uint8(v)
 	return base
 }
+
+var vtprotoPool_Object_Vector = sync.Pool{
+	New: func() interface{} {
+		return &Object_Vector{}
+	},
+}
+
+func (m *Object_Vector) ResetVT() {
+	f0 := m.Vector[:0]
+	m.Reset()
+	m.Vector = f0
+}
+func (m *Object_Vector) ReturnToVTPool() {
+	if m != nil {
+		m.ResetVT()
+		vtprotoPool_Object_Vector.Put(m)
+	}
+}
+func Object_VectorFromVTPool() *Object_Vector {
+	return vtprotoPool_Object_Vector.Get().(*Object_Vector)
+}
 func (m *Search_Request) SizeVT() (n int) {
 	if m == nil {
 		return 0
@@ -13780,7 +13802,7 @@ func (m *Object_Vector) UnmarshalVT(dAtA []byte) error {
 				}
 				var elementCount int
 				elementCount = packedLen / 4
-				if elementCount != 0 && len(m.Vector) == 0 {
+				if elementCount != 0 && len(m.Vector) == 0 && cap(m.Vector) < elementCount {
 					m.Vector = make([]float32, 0, elementCount)
 				}
 				for iNdEx < postIndex {

diff --git a/apis/grpc/v1/vald/object_vtproto.pb.go b/apis/grpc/v1/vald/object_vtproto.pb.go
@@ -70,7 +70,7 @@ func (c *objectClient) Exists(ctx context.Context, in *payload.Object_ID, opts .
 }
 
 func (c *objectClient) GetObject(ctx context.Context, in *payload.Object_VectorRequest, opts ...grpc.CallOption) (*payload.Object_Vector, error) {
-	out := new(payload.Object_Vector)
+	out := payload.Object_VectorFromVTPool()
 	err := c.cc.Invoke(ctx, "/vald.v1.Object/GetObject", in, out, opts...)
 	if err != nil {
 		return nil, err

diff --git a/apis/proto/v1/payload/payload.proto b/apis/proto/v1/payload/payload.proto
@@ -20,11 +20,13 @@ package payload.v1;
 
 import "github.com/envoyproxy/protoc-gen-validate/validate/validate.proto";
 import "github.com/googleapis/googleapis/google/rpc/status.proto";
+import "github.com/planetscale/vtprotobuf/include/github.com/planetscale/vtprotobuf/vtproto/ext.proto";
 
 option go_package = "github.com/vdaas/vald/apis/grpc/v1/payload";
 option java_multiple_files = true;
 option java_outer_classname = "ValdPayload";
 option java_package = "org.vdaas.vald.api.v1.payload";
+option objc_class_prefix = "Payload";
 
 // Search related messages.
 message Search {
@@ -367,6 +369,7 @@ message Object {
 
   // Represent a vector.
   message Vector {
+    option (vtproto.mempool) = true;
     // The vector ID.
     string id = 1 [(validate.rules).string.min_len = 1];
     // The vector.