enhance: Update protobuf-go to protobuf-go v2 (#34394)

issue: #34252 Signed-off-by: Wei Liu <[email protected]> Co-authored-by: Congqi Xia <[email protected]>
milvus-io · Aug 19, 2024 · 7e43d57 · 7e43d57
1 parent 14ec3dc
commit 7e43d57
Show file tree

Hide file tree

Showing 221 changed files with 1,471 additions and 1,388 deletions.
diff --git a/Makefile b/Makefile
@@ -60,6 +60,14 @@ INSTALL_GOFUMPT := $(findstring $(GOFUMPT_VERSION),$(GOFUMPT_OUTPUT))
 GOTESTSUM_VERSION := 1.11.0
 GOTESTSUM_OUTPUT := $(shell $(INSTALL_PATH)/gotestsum --version 2>/dev/null)
 INSTALL_GOTESTSUM := $(findstring $(GOTESTSUM_VERSION),$(GOTESTSUM_OUTPUT))
+# protoc-gen-go
+PROTOC_GEN_GO_VERSION := 1.33.0
+PROTOC_GEN_GO_OUTPUT := $(shell print | $(INSTALL_PATH)/protoc-gen-go --version 2>/dev/null)
+INSTALL_PROTOC_GEN_GO := $(findstring $(PROTOC_GEN_GO_VERSION),$(PROTOC_GEN_GO_OUTPUT))
+# protoc-gen-go-grpc
+PROTOC_GEN_GO_GRPC_VERSION := 1.3.0
+PROTOC_GEN_GO_GRPC_OUTPUT := $(shell print | $(INSTALL_PATH)/protoc-gen-go-grpc  --version 2>/dev/null)
+INSTALL_PROTOC_GEN_GO_GRPC := $(findstring $(PROTOC_GEN_GO_GRPC_VERSION),$(PROTOC_GEN_GO_GRPC_OUTPUT))
 
 index_engine = knowhere
 
@@ -105,6 +113,19 @@ getdeps:
 		echo "gotestsum v$(GOTESTSUM_VERSION) already installed";\
 	fi
 
+get-proto-deps:
+	@mkdir -p $(INSTALL_PATH) # make sure directory exists
+	@if [ -z "$(INSTALL_PROTOC_GEN_GO)" ]; then \
+		echo "install protoc-gen-go $(PROTOC_GEN_GO_VERSION) to $(INSTALL_PATH)" && GOBIN=$(INSTALL_PATH) go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(PROTOC_GEN_GO_VERSION); \
+	else \
+		echo "protoc-gen-go@v$(PROTOC_GEN_GO_VERSION) already installed";\
+	fi
+	@if [ -z "$(INSTALL_PROTOC_GEN_GO_GRPC)" ]; then \
+		echo "install protoc-gen-go-grpc $(PROTOC_GEN_GO_GRPC_VERSION) to $(INSTALL_PATH)" && GOBIN=$(INSTALL_PATH) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v$(PROTOC_GEN_GO_GRPC_VERSION); \
+	else \
+		echo "protoc-gen-go-grpc@v$(PROTOC_GEN_GO_GRPC_VERSION) already installed";\
+	fi
+
 tools/bin/revive: tools/check/go.mod
 	cd tools/check; \
 	$(GO) build -pgo=$(PGO_PATH)/default.pgo -o ../bin/revive github.com/mgechev/revive
@@ -218,17 +239,13 @@ build-3rdparty:
 	@echo "Build 3rdparty ..."
 	@(env bash $(PWD)/scripts/3rdparty_build.sh -o ${use_opendal})
 
-generated-proto-without-cpp: download-milvus-proto
+generated-proto-without-cpp: download-milvus-proto get-proto-deps
 	@echo "Generate proto ..."
-	@mkdir -p ${GOPATH}/bin
-	@which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/[email protected])
-	@(env bash $(PWD)/scripts/generate_proto.sh)
+	@(env bash $(PWD)/scripts/generate_proto.sh ${INSTALL_PATH})
 
-generated-proto: download-milvus-proto build-3rdparty
+generated-proto: download-milvus-proto build-3rdparty get-proto-deps
 	@echo "Generate proto ..."
-	@mkdir -p ${GOPATH}/bin
-	@which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/[email protected])
-	@(env bash $(PWD)/scripts/generate_proto.sh)
+	@(env bash $(PWD)/scripts/generate_proto.sh ${INSTALL_PATH})
 
 build-cpp: generated-proto
 	@echo "Building Milvus cpp library ..."

diff --git a/client/client.go b/client/client.go
@@ -23,9 +23,9 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/gogo/status"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"

diff --git a/client/collection_options.go b/client/collection_options.go
@@ -17,7 +17,7 @@
 package client
 
 import (
-	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"

diff --git a/client/collection_test.go b/client/collection_test.go
@@ -21,10 +21,10 @@ import (
 	"fmt"
 	"testing"
 
-	"github.com/golang/protobuf/proto"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"

diff --git a/client/go.mod b/client/go.mod
@@ -5,8 +5,6 @@ go 1.21
 require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/cockroachdb/errors v1.9.1
-	github.com/gogo/status v1.1.0
-	github.com/golang/protobuf v1.5.4
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/milvus-io/milvus-proto/go-api/v2 v2.4.9
 	github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3
@@ -16,6 +14,7 @@ require (
 	github.com/tidwall/gjson v1.17.1
 	go.uber.org/atomic v1.10.0
 	google.golang.org/grpc v1.57.1
+	google.golang.org/protobuf v1.33.0
 )
 
 require (
@@ -39,8 +38,8 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/godbus/dbus/v5 v5.0.4 // indirect
-	github.com/gogo/googleapis v1.4.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/btree v1.1.2 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
@@ -117,7 +116,6 @@ require (
 	google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.62.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect

diff --git a/client/go.sum b/client/go.sum
@@ -198,15 +198,13 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E
 github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
-github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
 github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA=
 github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
 github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

diff --git a/client/read_options.go b/client/read_options.go
@@ -20,7 +20,7 @@ import (
 	"encoding/json"
 	"strconv"
 
-	"github.com/golang/protobuf/proto"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"

diff --git a/cmd/milvus/mck.go b/cmd/milvus/mck.go
@@ -12,9 +12,9 @@ import (
 	"time"
 
 	"github.com/cockroachdb/errors"
-	"github.com/golang/protobuf/proto"
 	clientv3 "go.etcd.io/etcd/client/v3"
 	"go.uber.org/zap"
+	"google.golang.org/protobuf/proto"
 
 	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
 	etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
@@ -528,40 +528,40 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
 
 	switch header.Base.MsgType {
 	case commonpb.MsgType_LoadCollection:
-		loadReq := querypb.LoadCollectionRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.LoadCollectionRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "LoadCollectionRequest", err)
 		}
 		log.Info("LoadCollection", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "LoadCollection", emptyInt64(), emptyInt64(), nil
 	case commonpb.MsgType_LoadPartitions:
-		loadReq := querypb.LoadPartitionsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.LoadPartitionsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "LoadPartitionsRequest", err)
 		}
 		log.Info("LoadPartitions", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "LoadPartitions", loadReq.PartitionIDs, emptyInt64(), nil
 	case commonpb.MsgType_ReleaseCollection:
-		loadReq := querypb.ReleaseCollectionRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.ReleaseCollectionRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "ReleaseCollectionRequest", err)
 		}
 		log.Info("ReleaseCollection", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "ReleaseCollection", emptyInt64(), emptyInt64(), nil
 	case commonpb.MsgType_ReleasePartitions:
-		loadReq := querypb.ReleasePartitionsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.ReleasePartitionsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "ReleasePartitionsRequest", err)
 		}
 		log.Info("ReleasePartitions", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "ReleasePartitions", loadReq.PartitionIDs, emptyInt64(), nil
 	case commonpb.MsgType_LoadSegments:
-		loadReq := querypb.LoadSegmentsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.LoadSegmentsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "LoadSegmentsRequest", err)
 		}
@@ -584,16 +584,16 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
 		log.Info("LoadSegments", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "LoadSegments", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil
 	case commonpb.MsgType_ReleaseSegments:
-		loadReq := querypb.ReleaseSegmentsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.ReleaseSegmentsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "ReleaseSegmentsRequest", err)
 		}
 		log.Info("ReleaseSegments", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "ReleaseSegments", loadReq.PartitionIDs, loadReq.SegmentIDs, nil
 	case commonpb.MsgType_WatchDmChannels:
-		loadReq := querypb.WatchDmChannelsRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.WatchDmChannelsRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "WatchDmChannelsRequest", err)
 		}
@@ -619,16 +619,16 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
 		log.Warn("legacy WatchQueryChannels type found, ignore")
 		return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil
 	case commonpb.MsgType_LoadBalanceSegments:
-		loadReq := querypb.LoadBalanceRequest{}
-		err = proto.Unmarshal([]byte(t), &loadReq)
+		loadReq := &querypb.LoadBalanceRequest{}
+		err = proto.Unmarshal([]byte(t), loadReq)
 		if err != nil {
 			return errReturn(taskID, "LoadBalanceRequest", err)
 		}
 		log.Info("LoadBalanceSegments", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
 		return "LoadBalanceSegments", emptyInt64(), loadReq.SealedSegmentIDs, nil
 	case commonpb.MsgType_HandoffSegments:
-		handoffReq := querypb.HandoffSegmentsRequest{}
-		err = proto.Unmarshal([]byte(t), &handoffReq)
+		handoffReq := &querypb.HandoffSegmentsRequest{}
+		err = proto.Unmarshal([]byte(t), handoffReq)
 		if err != nil {
 			return errReturn(taskID, "HandoffSegmentsRequest", err)
 		}

diff --git a/cmd/tools/datameta/main.go b/cmd/tools/datameta/main.go
@@ -6,8 +6,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/golang/protobuf/proto"
 	"go.uber.org/zap"
+	"google.golang.org/protobuf/proto"
 
 	etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
 	"github.com/milvus-io/milvus/internal/proto/datapb"

diff --git a/cmd/tools/migration/backend/backup_header.go b/cmd/tools/migration/backend/backup_header.go
@@ -3,8 +3,6 @@ package backend
 import (
 	"encoding/json"
 
-	"github.com/golang/protobuf/proto"
-
 	"github.com/milvus-io/milvus/cmd/tools/migration/console"
 )
 
@@ -14,32 +12,6 @@ const (
 	BackupHeaderVersionV1 BackupHeaderVersion = iota
 )
 
-// BackupHeader stores etcd backup header information
-type BackupHeader struct {
-	// Version number for backup format
-	Version BackupHeaderVersion `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"`
-	// instance name, as rootPath for key prefix
-	Instance string `protobuf:"bytes,2,opt,name=instance,proto3" json:"instance,omitempty"`
-	// MetaPath used in keys
-	MetaPath string `protobuf:"bytes,3,opt,name=meta_path,proto3" json:"meta_path,omitempty"`
-	// Entries record number of key-value in backup
-	Entries int64 `protobuf:"varint,4,opt,name=entries,proto3" json:"entries,omitempty"`
-	// Component is the backup target
-	Component string `protobuf:"bytes,5,opt,name=component,proto3" json:"component,omitempty"`
-	// Extra property reserved
-	Extra []byte `protobuf:"bytes,6,opt,name=extra,proto3" json:"-"`
-}
-
-func (v *BackupHeader) Reset() {
-	*v = BackupHeader{}
-}
-
-func (v *BackupHeader) String() string {
-	return proto.CompactTextString(v)
-}
-
-func (v *BackupHeader) ProtoMessage() {}
-
 type BackupHeaderExtra struct {
 	EntryIncludeRootPath bool `json:"entry_include_root_path"`
 }