Skip to content

Commit

Permalink
enhance: Update protobuf-go to protobuf-go v2 (#34394) (#35555)
Browse files Browse the repository at this point in the history
issue: #34252
pr: #34394 #35072 #35084

Signed-off-by: Wei Liu <[email protected]>
Co-authored-by: Congqi Xia <[email protected]>
  • Loading branch information
weiliu1031 and congqixia authored Aug 21, 2024
1 parent bfd8758 commit e2542a1
Show file tree
Hide file tree
Showing 221 changed files with 1,583 additions and 1,394 deletions.
33 changes: 25 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,14 @@ INSTALL_GOFUMPT := $(findstring $(GOFUMPT_VERSION),$(GOFUMPT_OUTPUT))
GOTESTSUM_VERSION := 1.11.0
GOTESTSUM_OUTPUT := $(shell $(INSTALL_PATH)/gotestsum --version 2>/dev/null)
INSTALL_GOTESTSUM := $(findstring $(GOTESTSUM_VERSION),$(GOTESTSUM_OUTPUT))
# protoc-gen-go
PROTOC_GEN_GO_VERSION := 1.33.0
PROTOC_GEN_GO_OUTPUT := $(shell print | $(INSTALL_PATH)/protoc-gen-go --version 2>/dev/null)
INSTALL_PROTOC_GEN_GO := $(findstring $(PROTOC_GEN_GO_VERSION),$(PROTOC_GEN_GO_OUTPUT))
# protoc-gen-go-grpc
PROTOC_GEN_GO_GRPC_VERSION := 1.3.0
PROTOC_GEN_GO_GRPC_OUTPUT := $(shell print | $(INSTALL_PATH)/protoc-gen-go-grpc --version 2>/dev/null)
INSTALL_PROTOC_GEN_GO_GRPC := $(findstring $(PROTOC_GEN_GO_GRPC_VERSION),$(PROTOC_GEN_GO_GRPC_OUTPUT))

index_engine = knowhere

Expand Down Expand Up @@ -105,6 +113,19 @@ getdeps:
echo "gotestsum v$(GOTESTSUM_VERSION) already installed";\
fi

get-proto-deps:
@mkdir -p $(INSTALL_PATH) # make sure directory exists
@if [ -z "$(INSTALL_PROTOC_GEN_GO)" ]; then \
echo "install protoc-gen-go $(PROTOC_GEN_GO_VERSION) to $(INSTALL_PATH)" && GOBIN=$(INSTALL_PATH) go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(PROTOC_GEN_GO_VERSION); \
else \
echo "protoc-gen-go@v$(PROTOC_GEN_GO_VERSION) already installed";\
fi
@if [ -z "$(INSTALL_PROTOC_GEN_GO_GRPC)" ]; then \
echo "install protoc-gen-go-grpc $(PROTOC_GEN_GO_GRPC_VERSION) to $(INSTALL_PATH)" && GOBIN=$(INSTALL_PATH) go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v$(PROTOC_GEN_GO_GRPC_VERSION); \
else \
echo "protoc-gen-go-grpc@v$(PROTOC_GEN_GO_GRPC_VERSION) already installed";\
fi

tools/bin/revive: tools/check/go.mod
cd tools/check; \
$(GO) build -pgo=$(PGO_PATH)/default.pgo -o ../bin/revive github.com/mgechev/revive
Expand Down Expand Up @@ -218,17 +239,13 @@ build-3rdparty:
@echo "Build 3rdparty ..."
@(env bash $(PWD)/scripts/3rdparty_build.sh -o ${use_opendal})

generated-proto-without-cpp: download-milvus-proto
generated-proto-without-cpp: download-milvus-proto get-proto-deps
@echo "Generate proto ..."
@mkdir -p ${GOPATH}/bin
@which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/[email protected])
@(env bash $(PWD)/scripts/generate_proto.sh)
@(env bash $(PWD)/scripts/generate_proto.sh ${INSTALL_PATH})

generated-proto: download-milvus-proto build-3rdparty
generated-proto: download-milvus-proto build-3rdparty get-proto-deps
@echo "Generate proto ..."
@mkdir -p ${GOPATH}/bin
@which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/[email protected])
@(env bash $(PWD)/scripts/generate_proto.sh)
@(env bash $(PWD)/scripts/generate_proto.sh ${INSTALL_PATH})

build-cpp: generated-proto
@echo "Building Milvus cpp library ..."
Expand Down
2 changes: 1 addition & 1 deletion client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ import (
"strconv"
"time"

"github.com/gogo/status"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"

"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
Expand Down
2 changes: 1 addition & 1 deletion client/collection_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package client

import (
"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/proto"

"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
Expand Down
2 changes: 1 addition & 1 deletion client/collection_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ import (
"fmt"
"testing"

"github.com/golang/protobuf/proto"
"github.com/samber/lo"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"google.golang.org/protobuf/proto"

"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
Expand Down
8 changes: 3 additions & 5 deletions client/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,16 @@ go 1.21
require (
github.com/blang/semver/v4 v4.0.0
github.com/cockroachdb/errors v1.9.1
github.com/gogo/status v1.1.0
github.com/golang/protobuf v1.5.4
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.9
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240821072125-950375c251c1
github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3
github.com/quasilyte/go-ruleguard/dsl v0.3.22
github.com/samber/lo v1.27.0
github.com/stretchr/testify v1.8.4
github.com/tidwall/gjson v1.17.1
go.uber.org/atomic v1.10.0
google.golang.org/grpc v1.57.1
google.golang.org/protobuf v1.33.0
)

require (
Expand All @@ -39,8 +38,8 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/gogo/googleapis v1.4.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
Expand Down Expand Up @@ -117,7 +116,6 @@ require (
google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
Expand Down
6 changes: 2 additions & 4 deletions client/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -198,15 +198,13 @@ github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/E
github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/gogo/status v1.1.0 h1:+eIkrewn5q6b30y+g/BJINVVdi2xH7je5MPJ3ZPK3JA=
github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
Expand Down Expand Up @@ -402,8 +400,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.9 h1:GhdkA7L4hE/TpUc6CzC9QufoaaF7StLRnISLsXexVLo=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.9/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240821072125-950375c251c1 h1:2SVCzfGzDhmvT5FbAC38cil2MAY3awPUQFGsB87y7io=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.10-0.20240821072125-950375c251c1/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 h1:ZBpRWhBa7FTFxW4YYVv9AUESoW1Xyb3KNXTzTqfkZmw=
github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3/go.mod h1:jQ2BUZny1COsgv1Qbcv8dmbppW+V9J/c4YQZNb3EOm8=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
Expand Down
110 changes: 110 additions & 0 deletions client/mock_milvus_server_test.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion client/read_options.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
"encoding/json"
"strconv"

"github.com/golang/protobuf/proto"
"google.golang.org/protobuf/proto"

"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
Expand Down
38 changes: 19 additions & 19 deletions cmd/milvus/mck.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ import (
"time"

"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"

"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
Expand Down Expand Up @@ -528,40 +528,40 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e

switch header.Base.MsgType {
case commonpb.MsgType_LoadCollection:
loadReq := querypb.LoadCollectionRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
loadReq := &querypb.LoadCollectionRequest{}
err = proto.Unmarshal([]byte(t), loadReq)
if err != nil {
return errReturn(taskID, "LoadCollectionRequest", err)
}
log.Info("LoadCollection", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "LoadCollection", emptyInt64(), emptyInt64(), nil
case commonpb.MsgType_LoadPartitions:
loadReq := querypb.LoadPartitionsRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
loadReq := &querypb.LoadPartitionsRequest{}
err = proto.Unmarshal([]byte(t), loadReq)
if err != nil {
return errReturn(taskID, "LoadPartitionsRequest", err)
}
log.Info("LoadPartitions", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "LoadPartitions", loadReq.PartitionIDs, emptyInt64(), nil
case commonpb.MsgType_ReleaseCollection:
loadReq := querypb.ReleaseCollectionRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
loadReq := &querypb.ReleaseCollectionRequest{}
err = proto.Unmarshal([]byte(t), loadReq)
if err != nil {
return errReturn(taskID, "ReleaseCollectionRequest", err)
}
log.Info("ReleaseCollection", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "ReleaseCollection", emptyInt64(), emptyInt64(), nil
case commonpb.MsgType_ReleasePartitions:
loadReq := querypb.ReleasePartitionsRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
loadReq := &querypb.ReleasePartitionsRequest{}
err = proto.Unmarshal([]byte(t), loadReq)
if err != nil {
return errReturn(taskID, "ReleasePartitionsRequest", err)
}
log.Info("ReleasePartitions", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "ReleasePartitions", loadReq.PartitionIDs, emptyInt64(), nil
case commonpb.MsgType_LoadSegments:
loadReq := querypb.LoadSegmentsRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
loadReq := &querypb.LoadSegmentsRequest{}
err = proto.Unmarshal([]byte(t), loadReq)
if err != nil {
return errReturn(taskID, "LoadSegmentsRequest", err)
}
Expand All @@ -584,16 +584,16 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
log.Info("LoadSegments", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "LoadSegments", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil
case commonpb.MsgType_ReleaseSegments:
loadReq := querypb.ReleaseSegmentsRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
loadReq := &querypb.ReleaseSegmentsRequest{}
err = proto.Unmarshal([]byte(t), loadReq)
if err != nil {
return errReturn(taskID, "ReleaseSegmentsRequest", err)
}
log.Info("ReleaseSegments", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "ReleaseSegments", loadReq.PartitionIDs, loadReq.SegmentIDs, nil
case commonpb.MsgType_WatchDmChannels:
loadReq := querypb.WatchDmChannelsRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
loadReq := &querypb.WatchDmChannelsRequest{}
err = proto.Unmarshal([]byte(t), loadReq)
if err != nil {
return errReturn(taskID, "WatchDmChannelsRequest", err)
}
Expand All @@ -619,16 +619,16 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
log.Warn("legacy WatchQueryChannels type found, ignore")
return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil
case commonpb.MsgType_LoadBalanceSegments:
loadReq := querypb.LoadBalanceRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
loadReq := &querypb.LoadBalanceRequest{}
err = proto.Unmarshal([]byte(t), loadReq)
if err != nil {
return errReturn(taskID, "LoadBalanceRequest", err)
}
log.Info("LoadBalanceSegments", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "LoadBalanceSegments", emptyInt64(), loadReq.SealedSegmentIDs, nil
case commonpb.MsgType_HandoffSegments:
handoffReq := querypb.HandoffSegmentsRequest{}
err = proto.Unmarshal([]byte(t), &handoffReq)
handoffReq := &querypb.HandoffSegmentsRequest{}
err = proto.Unmarshal([]byte(t), handoffReq)
if err != nil {
return errReturn(taskID, "HandoffSegmentsRequest", err)
}
Expand Down
Loading

0 comments on commit e2542a1

Please sign in to comment.