From fea25795c207fe8890609247e38a4e2a740fb87b Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 27 Mar 2017 15:55:00 +0200 Subject: [PATCH 01/82] Initial grpc code --- glide.lock | 18 +++++---- glide.yaml | 2 + transport/x/grpc/doc.go | 22 +++++++++++ transport/x/grpc/inbound.go | 75 ++++++++++++++++++++++++++++++++++++ transport/x/grpc/outbound.go | 65 +++++++++++++++++++++++++++++++ 5 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 transport/x/grpc/doc.go create mode 100644 transport/x/grpc/inbound.go create mode 100644 transport/x/grpc/outbound.go diff --git a/glide.lock b/glide.lock index 3c554eb36..3b3ab1108 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 3756b8708cb1e31ddb382de6bdd3985edc6d68585e7916b086e4232c1d675093 -updated: 2017-03-24T17:57:40.071228437+01:00 +hash: 44198e8a5ee2afc575d30810c58619705b33ae48c047c801dbd6bbfac2093a4b +updated: 2017-03-27T15:47:50.121051691+02:00 imports: - name: github.com/apache/thrift version: b2a4d4ae21c789b689dd162deb819665567f481c @@ -94,7 +94,7 @@ imports: - name: github.com/uber-common/bark version: 8841a0f8e7ca869284ccb29c08a14cf3f4310f46 - name: github.com/uber-go/atomic - version: e682c1008ac17bf26d2e4b5ad6cdd08520ed0b22 + version: 3b8db5e93c4c02efbc313e17b2e796b0914a01fb - name: github.com/uber-go/tally version: 4b9d9de43ffcb0b7efe67dab2a92c2d58dbf16ab subpackages: @@ -103,7 +103,7 @@ imports: - m3/thrift - m3/thriftudp - name: github.com/uber/cherami-client-go - version: 1ef6560c59eeb170f2381c02287ca44a5c50a523 + version: 5b9071f590e7c32b88b950ffc2a4e20ef74d2804 subpackages: - client/cherami - common @@ -112,7 +112,7 @@ imports: - common/websocket - stream - name: github.com/uber/cherami-thrift - version: 09ed2ceaeab9e52820a81caece0dee9914c31f5d + version: 959013313a26bbb974e23286ef4b9991fde4947b subpackages: - .generated/go/cherami - name: github.com/uber/jaeger-client-go @@ -172,18 +172,20 @@ imports: subpackages: - update-license - name: golang.org/x/net - version: 3e967e1d28d2c9c06e749dc2bdc14b04df89e689 + version: 6c23252515492caf9b228a9d5cabcdbde29f7f82 subpackages: - context - context/ctxhttp - name: golang.org/x/sys - version: 99f16d856c9836c42d24e7ab64ea72916925fa97 + version: afadfcc7779c1f4db0f6f6438afcb108d9c9c7cd subpackages: - unix - name: golang.org/x/tools version: c21bc47f893ee73c18dd2119bb8ff9a2b492c4c6 subpackages: - go/ast/astutil +- name: google.golang.org/grpc + version: cdee119ee21e61eef7093a41ba148fa83585e143 - name: gopkg.in/redis.v5 version: a16aeec10ff407b1e7be6dd35797ccf5426ef0f0 subpackages: @@ -193,5 +195,5 @@ imports: - internal/pool - internal/proto - name: gopkg.in/yaml.v2 - version: a83829b6f1293c91addabc89d0571c246397bbf4 + version: a3f3340b5840cee44f372bddb5880fcbc419b46a testImports: [] diff --git a/glide.yaml b/glide.yaml index 84871fe57..f53fd97d4 100644 --- a/glide.yaml +++ b/glide.yaml @@ -19,6 +19,8 @@ import: version: ^1 - package: github.com/gogo/protobuf version: ~0.4 +- package: google.golang.org/grpc + version: ^1.2 - package: github.com/crossdock/crossdock-go version: master - package: github.com/golang/mock diff --git a/transport/x/grpc/doc.go b/transport/x/grpc/doc.go new file mode 100644 index 000000000..d460585d7 --- /dev/null +++ b/transport/x/grpc/doc.go @@ -0,0 +1,22 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package grpc implements the grpc transport. +package grpc diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go new file mode 100644 index 000000000..077cb3744 --- /dev/null +++ b/transport/x/grpc/inbound.go @@ -0,0 +1,75 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "go.uber.org/yarpc/api/transport" + "go.uber.org/yarpc/internal/sync" +) + +var _ transport.Inbound = (*Inbound)(nil) + +// Inbound is a grpc transport.Inbound. +type Inbound struct { + once sync.LifecycleOnce + address string + router transport.Router +} + +// NewInbound returns a new Inbound for the given address. +func NewInbound(address string) (*Inbound, error) { + return &Inbound{ + sync.Once(), + address, + nil, + }, nil +} + +// Start implements transport.Lifecycle#Start. +func (i *Inbound) Start() error { + return i.once.Start(i.start) +} + +// Stop implements transport.Lifecycle#Stop. +func (i *Inbound) Stop() error { + return i.once.Stop(i.stop) +} + +// IsRunning implements transport.Lifecycle#IsRunning. +func (i *Inbound) IsRunning() bool { + return i.IsRunning() +} + +// SetRouter implements transport.Inbound#SetRouter. +func (i *Inbound) SetRouter(router transport.Router) {} + +// Transports implements transport.Inbound#Transports. +func (i *Inbound) Transports() []transport.Transport { + return []transport.Transport{} +} + +func (i *Inbound) start() error { + return nil +} + +func (i *Inbound) stop() error { + return nil +} diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go new file mode 100644 index 000000000..72a83cea3 --- /dev/null +++ b/transport/x/grpc/outbound.go @@ -0,0 +1,65 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "context" + + "go.uber.org/yarpc/api/peer" + "go.uber.org/yarpc/api/transport" +) + +var _ transport.UnaryOutbound = (*Outbound)(nil) + +// Outbound is a transport.UnaryOutbound. +type Outbound struct { + chooser peer.Chooser +} + +// NewOutbound returns a new Outbound for the given chooser. +func NewOutbound(chooser peer.Chooser) (*Outbound, error) { + return nil, nil +} + +// Start implements transport.Lifecycle#Start. +func (o *Outbound) Start() error { + return nil +} + +// Stop implements transport.Lifecycle#Stop. +func (o *Outbound) Stop() error { + return nil +} + +// IsRunning implements transport.Lifecycle#IsRunning. +func (o *Outbound) IsRunning() bool { + return false +} + +// Transports implements transport.Inbound#Transports. +func (o *Outbound) Transports() []transport.Transport { + return nil +} + +// Call implements transport.UnaryOutbound#Call. +func (o *Outbound) Call(ctx context.Context, request *transport.Request) (*transport.Response, error) { + return nil, nil +} From 8ccec605d3f2a2f8f3ee1a4cdc40114651201efb Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 27 Mar 2017 16:03:13 +0200 Subject: [PATCH 02/82] More grpc work --- transport/x/grpc/inbound.go | 21 ++++++++++++--------- transport/x/grpc/outbound.go | 14 ++++++++------ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 077cb3744..37d8d0092 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -21,26 +21,25 @@ package grpc import ( + "sync" + "go.uber.org/yarpc/api/transport" - "go.uber.org/yarpc/internal/sync" + internalsync "go.uber.org/yarpc/internal/sync" ) var _ transport.Inbound = (*Inbound)(nil) // Inbound is a grpc transport.Inbound. type Inbound struct { - once sync.LifecycleOnce + once internalsync.LifecycleOnce address string router transport.Router + lock sync.RWMutex } // NewInbound returns a new Inbound for the given address. -func NewInbound(address string) (*Inbound, error) { - return &Inbound{ - sync.Once(), - address, - nil, - }, nil +func NewInbound(address string) *Inbound { + return &Inbound{internalsync.Once(), address, nil, sync.RWMutex{}} } // Start implements transport.Lifecycle#Start. @@ -59,7 +58,11 @@ func (i *Inbound) IsRunning() bool { } // SetRouter implements transport.Inbound#SetRouter. -func (i *Inbound) SetRouter(router transport.Router) {} +func (i *Inbound) SetRouter(router transport.Router) { + i.lock.Lock() + defer i.lock.Unlock() + i.router = router +} // Transports implements transport.Inbound#Transports. func (i *Inbound) Transports() []transport.Transport { diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 72a83cea3..50856687d 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -25,38 +25,40 @@ import ( "go.uber.org/yarpc/api/peer" "go.uber.org/yarpc/api/transport" + internalsync "go.uber.org/yarpc/internal/sync" ) var _ transport.UnaryOutbound = (*Outbound)(nil) // Outbound is a transport.UnaryOutbound. type Outbound struct { + once internalsync.LifecycleOnce chooser peer.Chooser } // NewOutbound returns a new Outbound for the given chooser. -func NewOutbound(chooser peer.Chooser) (*Outbound, error) { - return nil, nil +func NewOutbound(chooser peer.Chooser) *Outbound { + return &Outbound{internalsync.Once(), chooser} } // Start implements transport.Lifecycle#Start. func (o *Outbound) Start() error { - return nil + return o.once.Start(o.chooser.Start) } // Stop implements transport.Lifecycle#Stop. func (o *Outbound) Stop() error { - return nil + return o.once.Stop(o.chooser.Stop) } // IsRunning implements transport.Lifecycle#IsRunning. func (o *Outbound) IsRunning() bool { - return false + return o.once.IsRunning() } // Transports implements transport.Inbound#Transports. func (o *Outbound) Transports() []transport.Transport { - return nil + return []transport.Transport{} } // Call implements transport.UnaryOutbound#Call. From 840241af658be712603eeae5c3b2a85e868821b2 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 27 Mar 2017 16:05:02 +0200 Subject: [PATCH 03/82] More grpc work --- transport/x/grpc/inbound.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 37d8d0092..c082b0811 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -32,14 +32,14 @@ var _ transport.Inbound = (*Inbound)(nil) // Inbound is a grpc transport.Inbound. type Inbound struct { once internalsync.LifecycleOnce + lock sync.RWMutex address string router transport.Router - lock sync.RWMutex } // NewInbound returns a new Inbound for the given address. func NewInbound(address string) *Inbound { - return &Inbound{internalsync.Once(), address, nil, sync.RWMutex{}} + return &Inbound{internalsync.Once(), sync.RWMutex{}, address, nil} } // Start implements transport.Lifecycle#Start. From c0c98385ac3e9931de561bea696a1f23e874b665 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 27 Mar 2017 16:38:22 +0200 Subject: [PATCH 04/82] Copy in noopCodec from grpc branch --- transport/x/grpc/codec.go | 59 ++++++++++++++++++++++++++++++++++ transport/x/grpc/codec_test.go | 59 ++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 transport/x/grpc/codec.go create mode 100644 transport/x/grpc/codec_test.go diff --git a/transport/x/grpc/codec.go b/transport/x/grpc/codec.go new file mode 100644 index 000000000..5ff7176bf --- /dev/null +++ b/transport/x/grpc/codec.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import "fmt" + +var _bytesPointerInstance = &[]byte{} + +// noopCodec passes bytes to/from the wire without modification. +type noopCodec struct{} + +// Marshal takes a []byte pointer and passes it through as a []byte. +func (noopCodec) Marshal(value interface{}) ([]byte, error) { + bytesPointer, ok := value.(*[]byte) + if !ok { + return nil, newBytesPointerCastError(value) + } + return *bytesPointer, nil +} + +// Unmarshal takes a byte slice and writes it to v. +func (noopCodec) Unmarshal(data []byte, value interface{}) error { + bytesPointer, ok := value.(*[]byte) + if !ok { + return newBytesPointerCastError(value) + } + *bytesPointer = data + return nil +} + +func (noopCodec) String() string { + return "noop" +} + +func newBytesPointerCastError(actualObject interface{}) error { + return newCastError(_bytesPointerInstance, actualObject) +} + +func newCastError(expectedObjectOfType interface{}, actualObject interface{}) error { + return fmt.Errorf("expected object of type %T but got %T", expectedObjectOfType, actualObject) +} diff --git a/transport/x/grpc/codec_test.go b/transport/x/grpc/codec_test.go new file mode 100644 index 000000000..a078ce865 --- /dev/null +++ b/transport/x/grpc/codec_test.go @@ -0,0 +1,59 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPassThroughCodecMarshal(t *testing.T) { + value := []byte("test") + data, err := noopCodec{}.Marshal(&value) + assert.Equal(t, value, data) + assert.NoError(t, err) +} + +func TestPassThroughCodecMarshalError(t *testing.T) { + value := "test" + data, err := noopCodec{}.Marshal(&value) + assert.Equal(t, []byte(nil), data) + assert.Equal(t, newBytesPointerCastError(&value), err) +} + +func TestPassThroughCodecUnmarshal(t *testing.T) { + data := []byte("test") + var value []byte + assert.NoError(t, noopCodec{}.Unmarshal(data, &value)) + assert.Equal(t, data, value) +} + +func TestPassThroughCodecUnmarshalError(t *testing.T) { + var value string + err := noopCodec{}.Unmarshal([]byte("test"), &value) + assert.Equal(t, "", value) + assert.Equal(t, newBytesPointerCastError(&value), err) +} + +func TestPassThroughCodecString(t *testing.T) { + assert.Equal(t, "noop", noopCodec{}.String()) +} From f20daac7083854637e199972882a3c0ab607bcad Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 27 Mar 2017 16:52:07 +0200 Subject: [PATCH 05/82] More grpc work --- transport/x/grpc/inbound.go | 52 ++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index c082b0811..54e0f8f12 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -21,25 +21,34 @@ package grpc import ( + "errors" + "net" "sync" + "google.golang.org/grpc" + "go.uber.org/yarpc/api/transport" internalsync "go.uber.org/yarpc/internal/sync" ) -var _ transport.Inbound = (*Inbound)(nil) +var ( + errRouterNotSet = errors.New("router not set") + + _ transport.Inbound = (*Inbound)(nil) +) // Inbound is a grpc transport.Inbound. type Inbound struct { once internalsync.LifecycleOnce - lock sync.RWMutex + lock sync.Mutex address string router transport.Router + server *grpc.Server } // NewInbound returns a new Inbound for the given address. func NewInbound(address string) *Inbound { - return &Inbound{internalsync.Once(), sync.RWMutex{}, address, nil} + return &Inbound{internalsync.Once(), sync.Mutex{}, address, nil, nil} } // Start implements transport.Lifecycle#Start. @@ -70,9 +79,46 @@ func (i *Inbound) Transports() []transport.Transport { } func (i *Inbound) start() error { + i.lock.Lock() + defer i.lock.Unlock() + if i.router == nil { + return errRouterNotSet + } + server := grpc.NewServer(grpc.CustomCodec(noopCodec{})) + if err := registerRouter(server, i.router); err != nil { + return err + } + listener, err := net.Listen("tcp", i.address) + if err != nil { + return err + } + go func() { + // TODO there should be some mechanism to block here + // there is a race because the listener gets set in the grpc + // Server implementation and we should be able to block + // until Serve initialization is done + // + // It would be even better if we could do this outside the + // lock in i + // + // TODO Server always returns a non-nil error but should + // we do something with some or all errors? + _ = server.Serve(listener) + }() + i.server = server return nil } func (i *Inbound) stop() error { + i.lock.Lock() + defer i.lock.Unlock() + if i.server != nil { + i.server.GracefulStop() + } + return nil +} + +// TODO +func registerRouter(server *grpc.Server, router transport.Router) error { return nil } From 4af2e29c8a86184aef04b4e20b3585f57013b088 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 27 Mar 2017 18:21:13 +0200 Subject: [PATCH 06/82] More grpc work --- transport/x/grpc/inbound.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 54e0f8f12..5b30b00c3 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -85,7 +85,7 @@ func (i *Inbound) start() error { return errRouterNotSet } server := grpc.NewServer(grpc.CustomCodec(noopCodec{})) - if err := registerRouter(server, i.router); err != nil { + if err := registerProcedures(server, i.router.Procedures()); err != nil { return err } listener, err := net.Listen("tcp", i.address) @@ -119,6 +119,6 @@ func (i *Inbound) stop() error { } // TODO -func registerRouter(server *grpc.Server, router transport.Router) error { +func registerProcedures(server *grpc.Server, procedures []transport.Procedure) error { return nil } From 9cb61f4b5f5d8a5a7834ba6365b255a495e99c07 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 27 Mar 2017 19:23:16 +0200 Subject: [PATCH 07/82] More grpc work --- transport/x/grpc/inbound.go | 62 +++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 6 deletions(-) diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 5b30b00c3..ed3d83fdf 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -22,17 +22,21 @@ package grpc import ( "errors" + "fmt" "net" + "net/url" "sync" "google.golang.org/grpc" "go.uber.org/yarpc/api/transport" + internalprocedure "go.uber.org/yarpc/internal/procedure" internalsync "go.uber.org/yarpc/internal/sync" ) var ( - errRouterNotSet = errors.New("router not set") + errRouterNotSet = errors.New("router not set") + errRouterHasNoProcedures = errors.New("router has no procedures") _ transport.Inbound = (*Inbound)(nil) ) @@ -84,10 +88,14 @@ func (i *Inbound) start() error { if i.router == nil { return errRouterNotSet } - server := grpc.NewServer(grpc.CustomCodec(noopCodec{})) - if err := registerProcedures(server, i.router.Procedures()); err != nil { + serviceDescs, err := getServiceDescs(i.router) + if err != nil { return err } + server := grpc.NewServer(grpc.CustomCodec(noopCodec{})) + for _, serviceDesc := range serviceDescs { + server.RegisterService(serviceDesc, noopGrpcStruct{}) + } listener, err := net.Listen("tcp", i.address) if err != nil { return err @@ -118,7 +126,49 @@ func (i *Inbound) stop() error { return nil } -// TODO -func registerProcedures(server *grpc.Server, procedures []transport.Procedure) error { - return nil +func getServiceDescs(router transport.Router) ([]*grpc.ServiceDesc, error) { + // TODO: router.Procedures() is not guaranteed to be immutable + procedures := router.Procedures() + if len(procedures) == 0 { + return nil, errRouterHasNoProcedures + } + serviceNameToServiceDesc := make(map[string]*grpc.ServiceDesc) + for _, procedure := range procedures { + serviceName, methodDesc, err := getServiceNameAndMethodDesc(router, procedure) + if err != nil { + return nil, err + } + serviceDesc, ok := serviceNameToServiceDesc[serviceName] + if !ok { + serviceDesc = &grpc.ServiceDesc{ + ServiceName: serviceName, + HandlerType: (*noopGrpcInterface)(nil), + } + serviceNameToServiceDesc[serviceName] = serviceDesc + } + serviceDesc.Methods = append(serviceDesc.Methods, methodDesc) + } + serviceDescs := make([]*grpc.ServiceDesc, 0, len(serviceNameToServiceDesc)) + for _, serviceDesc := range serviceNameToServiceDesc { + serviceDescs = append(serviceDescs, serviceDesc) + } + return serviceDescs, nil +} + +func getServiceNameAndMethodDesc(router transport.Router, procedure transport.Procedure) (string, grpc.MethodDesc, error) { + serviceName, methodName := internalprocedure.FromName(procedure.Name) + if serviceName == "" || methodName == "" { + return "", grpc.MethodDesc{}, fmt.Errorf("invalid procedure name: %s", procedure.Name) + } + // TODO: do we really need to do url.QueryEscape? + // Are there consequences if there is a diff from the string and the url.QueryEscape string? + serviceName = url.QueryEscape(serviceName) + methodName = url.QueryEscape(methodName) + return serviceName, grpc.MethodDesc{ + MethodName: methodName, + Handler: newMethodHandler(serviceName, methodName, router).Handle, + }, nil } + +type noopGrpcInterface interface{} +type noopGrpcStruct struct{} From 87d286f510414bf614be03cea07adfb4193c8b7d Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 27 Mar 2017 19:37:21 +0200 Subject: [PATCH 08/82] More grpc work --- transport/x/grpc/handlers.go | 54 ++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 transport/x/grpc/handlers.go diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go new file mode 100644 index 000000000..5de88c267 --- /dev/null +++ b/transport/x/grpc/handlers.go @@ -0,0 +1,54 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "fmt" + + "go.uber.org/yarpc/api/transport" + + "golang.org/x/net/context" + "google.golang.org/grpc" +) + +type methodHandler struct { + serviceName string + methodName string + router transport.Router +} + +func newMethodHandler(serviceName string, methodName string, router transport.Router) *methodHandler { + return &methodHandler{serviceName, methodName, router} +} + +func (m *methodHandler) Handle( + server interface{}, + ctx context.Context, + decodeFunc func(interface{}) error, + interceptor grpc.UnaryServerInterceptor, +) (interface{}, error) { + var data []byte + if err := decodeFunc(&data); err != nil { + return nil, err + } + fmt.Printf("%s\n%s\n%+v\n%s\n", m.serviceName, m.methodName, ctx, string(data)) + return nil, nil +} From 8fdfd515b582ac1c6e4d4d7defe9a07c2a318f8c Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 27 Mar 2017 21:42:22 +0200 Subject: [PATCH 09/82] Fix golint --- Makefile | 3 ++- transport/x/grpc/handlers.go | 2 +- transport/x/grpc/inbound.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 243973618..9865725f8 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ # Paths besides auto-detected generated files that should be excluded from # lint results. -LINT_EXCLUDES_EXTRAS = +LINT_EXCLUDES_EXTRAS = \ + transport/x/grpc/handlers.go # Regex for 'go vet' rules to ignore GOVET_IGNORE_RULES = \ diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index 5de88c267..91c92286a 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -39,7 +39,7 @@ func newMethodHandler(serviceName string, methodName string, router transport.Ro return &methodHandler{serviceName, methodName, router} } -func (m *methodHandler) Handle( +func (m *methodHandler) handle( server interface{}, ctx context.Context, decodeFunc func(interface{}) error, diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index ed3d83fdf..35ea15012 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -166,7 +166,7 @@ func getServiceNameAndMethodDesc(router transport.Router, procedure transport.Pr methodName = url.QueryEscape(methodName) return serviceName, grpc.MethodDesc{ MethodName: methodName, - Handler: newMethodHandler(serviceName, methodName, router).Handle, + Handler: newMethodHandler(serviceName, methodName, router).handle, }, nil } From 82a0b0284452ed10f3fcd119d8018ce2fbc1b343 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 28 Mar 2017 14:07:51 +0200 Subject: [PATCH 10/82] More grpc work --- .dockerignore | 3 +++ .gitignore | 3 +++ transport/x/grpc/inbound.go | 1 + transport/x/grpc/outbound.go | 46 +++++++++++++++++++++++++++++------- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/.dockerignore b/.dockerignore index 8ca7b2db5..74528c584 100644 --- a/.dockerignore +++ b/.dockerignore @@ -23,3 +23,6 @@ internal/examples/thrift-hello/hello/hello internal/examples/thrift-keyvalue/keyvalue/client/client internal/examples/thrift-keyvalue/keyvalue/server/server internal/examples/thrift-oneway/thrift-oneway + +# temporary +.grpc diff --git a/.gitignore b/.gitignore index 900e8106d..9c70a6ec4 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ /internal/examples/thrift-keyvalue/keyvalue/client/client /internal/examples/thrift-keyvalue/keyvalue/server/server /internal/examples/thrift-oneway/thrift-oneway + +# temporary +.grpc diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 35ea15012..149b065fc 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -92,6 +92,7 @@ func (i *Inbound) start() error { if err != nil { return err } + // TODO: want to support default codec server := grpc.NewServer(grpc.CustomCodec(noopCodec{})) for _, serviceDesc := range serviceDescs { server.RegisterService(serviceDesc, noopGrpcStruct{}) diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 50856687d..18334c6e5 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -22,8 +22,10 @@ package grpc import ( "context" + "sync" + + "google.golang.org/grpc" - "go.uber.org/yarpc/api/peer" "go.uber.org/yarpc/api/transport" internalsync "go.uber.org/yarpc/internal/sync" ) @@ -32,23 +34,25 @@ var _ transport.UnaryOutbound = (*Outbound)(nil) // Outbound is a transport.UnaryOutbound. type Outbound struct { - once internalsync.LifecycleOnce - chooser peer.Chooser + once internalsync.LifecycleOnce + lock sync.Mutex + address string + clientConn *grpc.ClientConn } -// NewOutbound returns a new Outbound for the given chooser. -func NewOutbound(chooser peer.Chooser) *Outbound { - return &Outbound{internalsync.Once(), chooser} +// NewSingleOutbound returns a new Outbound for the given adrress. +func NewSingleOutbound(address string) *Outbound { + return &Outbound{internalsync.Once(), sync.Mutex{}, address, nil} } // Start implements transport.Lifecycle#Start. func (o *Outbound) Start() error { - return o.once.Start(o.chooser.Start) + return o.once.Start(o.start) } // Stop implements transport.Lifecycle#Stop. func (o *Outbound) Stop() error { - return o.once.Stop(o.chooser.Stop) + return o.once.Stop(o.stop) } // IsRunning implements transport.Lifecycle#IsRunning. @@ -65,3 +69,29 @@ func (o *Outbound) Transports() []transport.Transport { func (o *Outbound) Call(ctx context.Context, request *transport.Request) (*transport.Response, error) { return nil, nil } + +func (o *Outbound) start() error { + // TODO: redial + clientConn, err := grpc.Dial( + o.address, + grpc.WithInsecure(), + // TODO: want to support default codec + grpc.WithCodec(noopCodec{}), + ) + if err != nil { + return err + } + o.lock.Lock() + defer o.lock.Unlock() + o.clientConn = clientConn + return nil +} + +func (o *Outbound) stop() error { + o.lock.Lock() + defer o.lock.Unlock() + if o.clientConn != nil { + return o.clientConn.Close() + } + return nil +} From 48c4329066170450eed119e8a3764725aade2736 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 28 Mar 2017 14:23:58 +0200 Subject: [PATCH 11/82] Custom codec --- transport/x/grpc/codec.go | 59 ++++++++++++++++++++-------------- transport/x/grpc/codec_test.go | 46 +++++++++++++++++++------- transport/x/grpc/inbound.go | 2 +- transport/x/grpc/outbound.go | 2 +- 4 files changed, 70 insertions(+), 39 deletions(-) diff --git a/transport/x/grpc/codec.go b/transport/x/grpc/codec.go index 5ff7176bf..c79e12468 100644 --- a/transport/x/grpc/codec.go +++ b/transport/x/grpc/codec.go @@ -20,40 +20,49 @@ package grpc -import "fmt" +import ( + "fmt" -var _bytesPointerInstance = &[]byte{} + "github.com/gogo/protobuf/proto" +) -// noopCodec passes bytes to/from the wire without modification. -type noopCodec struct{} +// customCodec will either handle proto.Message objects, or +// pass bytes to/from the wire without modification. +type customCodec struct{} -// Marshal takes a []byte pointer and passes it through as a []byte. -func (noopCodec) Marshal(value interface{}) ([]byte, error) { - bytesPointer, ok := value.(*[]byte) - if !ok { - return nil, newBytesPointerCastError(value) +// Marshal takes a proto.Message and marshals it, or +// takes a []byte pointer and passes it through as a []byte. +func (customCodec) Marshal(obj interface{}) ([]byte, error) { + switch value := obj.(type) { + case proto.Message: + return proto.Marshal(value) + case *[]byte: + return *value, nil + default: + return nil, newCustomCodecCastError(obj) } - return *bytesPointer, nil } -// Unmarshal takes a byte slice and writes it to v. -func (noopCodec) Unmarshal(data []byte, value interface{}) error { - bytesPointer, ok := value.(*[]byte) - if !ok { - return newBytesPointerCastError(value) +// Unmarshal takes a proto.Message and unmarshals it, or +// takes a []byte pointer and writes it to v. +func (customCodec) Unmarshal(data []byte, obj interface{}) error { + switch value := obj.(type) { + case proto.Message: + return proto.Unmarshal(data, value) + case *[]byte: + *value = data + return nil + default: + return newCustomCodecCastError(obj) } - *bytesPointer = data - return nil } -func (noopCodec) String() string { - return "noop" +func (customCodec) String() string { + // TODO: we might have to fake this as proto + // this is hacky + return "custom" } -func newBytesPointerCastError(actualObject interface{}) error { - return newCastError(_bytesPointerInstance, actualObject) -} - -func newCastError(expectedObjectOfType interface{}, actualObject interface{}) error { - return fmt.Errorf("expected object of type %T but got %T", expectedObjectOfType, actualObject) +func newCustomCodecCastError(actualObject interface{}) error { + return fmt.Errorf("expected object of either type proto.Message or *[]byte but got %T", actualObject) } diff --git a/transport/x/grpc/codec_test.go b/transport/x/grpc/codec_test.go index a078ce865..baef9b4eb 100644 --- a/transport/x/grpc/codec_test.go +++ b/transport/x/grpc/codec_test.go @@ -23,37 +23,59 @@ package grpc import ( "testing" + "go.uber.org/yarpc/yarpcproto" + + "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/assert" ) -func TestPassThroughCodecMarshal(t *testing.T) { +func TestCustomCodecMarshalBytes(t *testing.T) { value := []byte("test") - data, err := noopCodec{}.Marshal(&value) + data, err := customCodec{}.Marshal(&value) assert.Equal(t, value, data) assert.NoError(t, err) } -func TestPassThroughCodecMarshalError(t *testing.T) { +func TestCustomCodecMarshalProtoMessage(t *testing.T) { + value := &yarpcproto.Oneway{true} + expectedData, err := proto.Marshal(value) + assert.NoError(t, err) + data, err := customCodec{}.Marshal(value) + assert.Equal(t, expectedData, data) + assert.NoError(t, err) +} + +func TestCustomCodecMarshalCastError(t *testing.T) { value := "test" - data, err := noopCodec{}.Marshal(&value) + data, err := customCodec{}.Marshal(&value) assert.Equal(t, []byte(nil), data) - assert.Equal(t, newBytesPointerCastError(&value), err) + assert.Equal(t, newCustomCodecCastError(&value), err) } -func TestPassThroughCodecUnmarshal(t *testing.T) { +func TestCustomCodecUnmarshalBytes(t *testing.T) { data := []byte("test") var value []byte - assert.NoError(t, noopCodec{}.Unmarshal(data, &value)) + assert.NoError(t, customCodec{}.Unmarshal(data, &value)) assert.Equal(t, data, value) } -func TestPassThroughCodecUnmarshalError(t *testing.T) { +func TestCustomCodecUnmarshalProtoMessage(t *testing.T) { + data := []byte("test") + expectedValue := &yarpcproto.Oneway{true} + data, err := proto.Marshal(expectedValue) + assert.NoError(t, err) + value := &yarpcproto.Oneway{} + assert.NoError(t, customCodec{}.Unmarshal(data, value)) + assert.Equal(t, expectedValue, value) +} + +func TestCustomCodecUnmarshalCastError(t *testing.T) { var value string - err := noopCodec{}.Unmarshal([]byte("test"), &value) + err := customCodec{}.Unmarshal([]byte("test"), &value) assert.Equal(t, "", value) - assert.Equal(t, newBytesPointerCastError(&value), err) + assert.Equal(t, newCustomCodecCastError(&value), err) } -func TestPassThroughCodecString(t *testing.T) { - assert.Equal(t, "noop", noopCodec{}.String()) +func TestCustomCodecString(t *testing.T) { + assert.Equal(t, "custom", customCodec{}.String()) } diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 149b065fc..74fc12e2d 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -93,7 +93,7 @@ func (i *Inbound) start() error { return err } // TODO: want to support default codec - server := grpc.NewServer(grpc.CustomCodec(noopCodec{})) + server := grpc.NewServer(grpc.CustomCodec(customCodec{})) for _, serviceDesc := range serviceDescs { server.RegisterService(serviceDesc, noopGrpcStruct{}) } diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 18334c6e5..c0feaad2c 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -76,7 +76,7 @@ func (o *Outbound) start() error { o.address, grpc.WithInsecure(), // TODO: want to support default codec - grpc.WithCodec(noopCodec{}), + grpc.WithCodec(customCodec{}), ) if err != nil { return err From 77acfecc453b76e162024dc96b60e308ceb411d4 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 11:31:40 +0200 Subject: [PATCH 12/82] Generate grpc for protobuf example --- .../examples/protobuf/examplepb/example.pb.go | 177 +++++++++++++++++- scripts/generate.sh | 23 ++- 2 files changed, 194 insertions(+), 6 deletions(-) diff --git a/internal/examples/protobuf/examplepb/example.pb.go b/internal/examples/protobuf/examplepb/example.pb.go index 90073910b..66780b7a2 100644 --- a/internal/examples/protobuf/examplepb/example.pb.go +++ b/internal/examples/protobuf/examplepb/example.pb.go @@ -40,11 +40,16 @@ package examplepb import proto "github.com/gogo/protobuf/proto" import fmt "fmt" import math "math" -import _ "go.uber.org/yarpc/yarpcproto" +import uber_yarpc "go.uber.org/yarpc/yarpcproto" import strings "strings" import reflect "reflect" +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -348,6 +353,176 @@ func valueToGoStringExample(v interface{}, typ string) string { pv := reflect.Indirect(rv).Interface() return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) } + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for KeyValue service + +type KeyValueClient interface { + GetValue(ctx context.Context, in *GetValueRequest, opts ...grpc.CallOption) (*GetValueResponse, error) + SetValue(ctx context.Context, in *SetValueRequest, opts ...grpc.CallOption) (*SetValueResponse, error) +} + +type keyValueClient struct { + cc *grpc.ClientConn +} + +func NewKeyValueClient(cc *grpc.ClientConn) KeyValueClient { + return &keyValueClient{cc} +} + +func (c *keyValueClient) GetValue(ctx context.Context, in *GetValueRequest, opts ...grpc.CallOption) (*GetValueResponse, error) { + out := new(GetValueResponse) + err := grpc.Invoke(ctx, "/uber.yarpc.internal.examples.protobuf.example.KeyValue/GetValue", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *keyValueClient) SetValue(ctx context.Context, in *SetValueRequest, opts ...grpc.CallOption) (*SetValueResponse, error) { + out := new(SetValueResponse) + err := grpc.Invoke(ctx, "/uber.yarpc.internal.examples.protobuf.example.KeyValue/SetValue", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for KeyValue service + +type KeyValueServer interface { + GetValue(context.Context, *GetValueRequest) (*GetValueResponse, error) + SetValue(context.Context, *SetValueRequest) (*SetValueResponse, error) +} + +func RegisterKeyValueServer(s *grpc.Server, srv KeyValueServer) { + s.RegisterService(&_KeyValue_serviceDesc, srv) +} + +func _KeyValue_GetValue_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetValueRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(KeyValueServer).GetValue(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/uber.yarpc.internal.examples.protobuf.example.KeyValue/GetValue", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(KeyValueServer).GetValue(ctx, req.(*GetValueRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _KeyValue_SetValue_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetValueRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(KeyValueServer).SetValue(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/uber.yarpc.internal.examples.protobuf.example.KeyValue/SetValue", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(KeyValueServer).SetValue(ctx, req.(*SetValueRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _KeyValue_serviceDesc = grpc.ServiceDesc{ + ServiceName: "uber.yarpc.internal.examples.protobuf.example.KeyValue", + HandlerType: (*KeyValueServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetValue", + Handler: _KeyValue_GetValue_Handler, + }, + { + MethodName: "SetValue", + Handler: _KeyValue_SetValue_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "internal/examples/protobuf/examplepb/example.proto", +} + +// Client API for Sink service + +type SinkClient interface { + Fire(ctx context.Context, in *FireRequest, opts ...grpc.CallOption) (*uber_yarpc.Oneway, error) +} + +type sinkClient struct { + cc *grpc.ClientConn +} + +func NewSinkClient(cc *grpc.ClientConn) SinkClient { + return &sinkClient{cc} +} + +func (c *sinkClient) Fire(ctx context.Context, in *FireRequest, opts ...grpc.CallOption) (*uber_yarpc.Oneway, error) { + out := new(uber_yarpc.Oneway) + err := grpc.Invoke(ctx, "/uber.yarpc.internal.examples.protobuf.example.Sink/Fire", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for Sink service + +type SinkServer interface { + Fire(context.Context, *FireRequest) (*uber_yarpc.Oneway, error) +} + +func RegisterSinkServer(s *grpc.Server, srv SinkServer) { + s.RegisterService(&_Sink_serviceDesc, srv) +} + +func _Sink_Fire_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FireRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SinkServer).Fire(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/uber.yarpc.internal.examples.protobuf.example.Sink/Fire", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SinkServer).Fire(ctx, req.(*FireRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Sink_serviceDesc = grpc.ServiceDesc{ + ServiceName: "uber.yarpc.internal.examples.protobuf.example.Sink", + HandlerType: (*SinkServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Fire", + Handler: _Sink_Fire_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "internal/examples/protobuf/examplepb/example.proto", +} + func (m *GetValueRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) diff --git a/scripts/generate.sh b/scripts/generate.sh index 4216ec938..0e0fcb4a8 100755 --- a/scripts/generate.sh +++ b/scripts/generate.sh @@ -25,16 +25,29 @@ generate_stringer() { # # $1: plugin # $2: file +# $3: other options protoc_with_imports() { protoc \ -I "${GOPATH}/src" \ -I vendor \ -I vendor/github.com/gogo/protobuf/protobuf \ -I . \ - "--${1}_out=Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:." \ + "--${1}_out=${3}Mgoogle/protobuf/descriptor.proto=github.com/gogo/protobuf/protoc-gen-gogo/descriptor,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:." \ "${2}" } +protoc_go() { + protoc_with_imports "gogoslick" "${1}" "" +} + +protoc_go_grpc() { + protoc_with_imports "gogoslick" "${1}" "plugins=grpc," +} + +protoc_yarpc_go() { + protoc_with_imports "yarpc-go" "${1}" "" +} + mockgen -destination=api/middleware/middlewaretest/router.go -package=middlewaretest go.uber.org/yarpc/api/middleware Router mockgen -destination=api/peer/peertest/list.go -package=peertest go.uber.org/yarpc/api/peer Chooser,List,ChooserList mockgen -destination=api/peer/peertest/peer.go -package=peertest go.uber.org/yarpc/api/peer Identifier,Peer @@ -70,10 +83,10 @@ thrift-gen --generateThrift --outputDir internal/crossdock/thrift/gen-go --input thrift --gen go:thrift_import=github.com/apache/thrift/lib/go/thrift --out internal/crossdock/thrift/gen-go internal/crossdock/thrift/gauntlet_apache.thrift -protoc_with_imports gogoslick yarpcproto/yarpc.proto -protoc_with_imports gogoslick encoding/x/protobuf/internal/wirepb/wire.proto -protoc_with_imports gogoslick internal/examples/protobuf/examplepb/example.proto -protoc_with_imports yarpc-go internal/examples/protobuf/examplepb/example.proto +protoc_go yarpcproto/yarpc.proto +protoc_go encoding/x/protobuf/internal/wirepb/wire.proto +protoc_go_grpc internal/examples/protobuf/examplepb/example.proto +protoc_yarpc_go internal/examples/protobuf/examplepb/example.proto touch encoding/x/protobuf/internal/wirepb/.nocover touch internal/crossdock/thrift/gen-go/echo/.nocover From 904c1f29a2b846aab724c287ec79a19c17837bf9 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 11:41:16 +0200 Subject: [PATCH 13/82] Add TransportTypeGRPC --- encoding/x/protobuf/testing/benchmark_test.go | 3 +++ encoding/x/protobuf/testing/testing_test.go | 3 +++ internal/testutils/testutils.go | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) diff --git a/encoding/x/protobuf/testing/benchmark_test.go b/encoding/x/protobuf/testing/benchmark_test.go index 0035ab264..da062bdc2 100644 --- a/encoding/x/protobuf/testing/benchmark_test.go +++ b/encoding/x/protobuf/testing/benchmark_test.go @@ -30,6 +30,9 @@ import ( func BenchmarkIntegration(b *testing.B) { for _, transportType := range testutils.AllTransportTypes { + if transportType == testutils.TransportTypeGRPC { + continue + } b.Run(transportType.String(), func(b *testing.B) { benchmarkIntegrationForTransportType(b, transportType) }) } } diff --git a/encoding/x/protobuf/testing/testing_test.go b/encoding/x/protobuf/testing/testing_test.go index 72c166f4c..1b4c6e307 100644 --- a/encoding/x/protobuf/testing/testing_test.go +++ b/encoding/x/protobuf/testing/testing_test.go @@ -36,6 +36,9 @@ import ( func TestIntegration(t *testing.T) { t.Parallel() for _, transportType := range testutils.AllTransportTypes { + if transportType == testutils.TransportTypeGRPC { + continue + } transportType := transportType t.Run(transportType.String(), func(t *testing.T) { testIntegrationForTransportType(t, transportType) }) } diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index afb8602b3..fa6247c46 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -30,6 +30,7 @@ import ( "go.uber.org/yarpc/internal/errors" "go.uber.org/yarpc/transport/http" "go.uber.org/yarpc/transport/tchannel" + "go.uber.org/yarpc/transport/x/grpc" ) const ( @@ -37,6 +38,8 @@ const ( TransportTypeHTTP TransportType = iota // TransportTypeTChannel represents using TChannel. TransportTypeTChannel + // TransportTypeGRPC represents using GRPC. + TransportTypeGRPC ) var ( @@ -44,6 +47,7 @@ var ( AllTransportTypes = []TransportType{ TransportTypeHTTP, TransportTypeTChannel, + TransportTypeGRPC, } ) @@ -57,6 +61,8 @@ func (t TransportType) String() string { return "http" case TransportTypeTChannel: return "tchannel" + case TransportTypeGRPC: + return "grpc" default: return strconv.Itoa(int(t)) } @@ -69,6 +75,8 @@ func ParseTransportType(s string) (TransportType, error) { return TransportTypeHTTP, nil case "tchannel": return TransportTypeTChannel, nil + case "grpc": + return TransportTypeGRPC, nil default: return 0, fmt.Errorf("invalid TransportType: %s", s) } @@ -130,6 +138,9 @@ func NewClientDispatcher(transportType TransportType, config *DispatcherConfig) httpOutbound := http.NewTransport().NewSingleOutbound(fmt.Sprintf("http://127.0.0.1:%d", port)) onewayOutbound = httpOutbound unaryOutbound = httpOutbound + case TransportTypeGRPC: + onewayOutbound = http.NewTransport().NewSingleOutbound(fmt.Sprintf("http://127.0.0.1:%d", httpPort)) + unaryOutbound = grpc.NewSingleOutbound(fmt.Sprintf("http://127.0.0.1:%d", port)) default: return nil, fmt.Errorf("invalid TransportType: %v", transportType) } @@ -156,6 +167,10 @@ func NewServerDispatcher(procedures []transport.Procedure, config *DispatcherCon if err != nil { return nil, err } + grpcPort, err := config.GetPort(TransportTypeGRPC) + if err != nil { + return nil, err + } tchannelTransport, err := tchannel.NewChannelTransport( tchannel.ServiceName(config.GetServiceName()), tchannel.ListenAddr(fmt.Sprintf(":%d", tchannelPort)), @@ -169,6 +184,7 @@ func NewServerDispatcher(procedures []transport.Procedure, config *DispatcherCon Inbounds: yarpc.Inbounds{ tchannelTransport.NewInbound(), http.NewTransport().NewInbound(fmt.Sprintf(":%d", httpPort)), + grpc.NewInbound(fmt.Sprintf(":%d", grpcPort)), }, }, ) From 30740b3f65a8808c46b1c0683fa06061afd9558b Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 12:09:47 +0200 Subject: [PATCH 14/82] Add grpc ClientConn to testutils --- encoding/x/protobuf/testing/benchmark_test.go | 4 +-- encoding/x/protobuf/testing/testing_test.go | 4 +-- internal/examples/protobuf/example/example.go | 22 ++++++++++++---- internal/examples/protobuf/main.go | 13 +++++----- internal/testutils/testutils.go | 25 +++++++++++++++++-- 5 files changed, 50 insertions(+), 18 deletions(-) diff --git a/encoding/x/protobuf/testing/benchmark_test.go b/encoding/x/protobuf/testing/benchmark_test.go index da062bdc2..956be882b 100644 --- a/encoding/x/protobuf/testing/benchmark_test.go +++ b/encoding/x/protobuf/testing/benchmark_test.go @@ -44,8 +44,8 @@ func benchmarkIntegrationForTransportType(b *testing.B, transportType testutils. transportType, keyValueYarpcServer, sinkYarpcServer, - func(keyValueYarpcClient examplepb.KeyValueYarpcClient, sinkYarpcClient examplepb.SinkYarpcClient) error { - benchmarkIntegration(b, keyValueYarpcClient, sinkYarpcClient, keyValueYarpcServer, sinkYarpcServer) + func(clients *example.Clients) error { + benchmarkIntegration(b, clients.KeyValueYarpcClient, clients.SinkYarpcClient, keyValueYarpcServer, sinkYarpcServer) return nil }, ) diff --git a/encoding/x/protobuf/testing/testing_test.go b/encoding/x/protobuf/testing/testing_test.go index 1b4c6e307..94291da5a 100644 --- a/encoding/x/protobuf/testing/testing_test.go +++ b/encoding/x/protobuf/testing/testing_test.go @@ -53,8 +53,8 @@ func testIntegrationForTransportType(t *testing.T, transportType testutils.Trans transportType, keyValueYarpcServer, sinkYarpcServer, - func(keyValueYarpcClient examplepb.KeyValueYarpcClient, sinkYarpcClient examplepb.SinkYarpcClient) error { - testIntegration(t, keyValueYarpcClient, sinkYarpcClient, keyValueYarpcServer, sinkYarpcServer) + func(clients *example.Clients) error { + testIntegration(t, clients.KeyValueYarpcClient, clients.SinkYarpcClient, keyValueYarpcServer, sinkYarpcServer) return nil }, ), diff --git a/internal/examples/protobuf/example/example.go b/internal/examples/protobuf/example/example.go index e86b0aeb1..e70dd3918 100644 --- a/internal/examples/protobuf/example/example.go +++ b/internal/examples/protobuf/example/example.go @@ -43,12 +43,20 @@ var ( errRequestValueNil = errors.New("request value nil") ) +// Clients holds all clients. +type Clients struct { + KeyValueYarpcClient examplepb.KeyValueYarpcClient + SinkYarpcClient examplepb.SinkYarpcClient + KeyGRPCClient examplepb.KeyValueClient + SinkGRPCClient examplepb.SinkClient +} + // WithClients calls f on the Clients. func WithClients( transportType testutils.TransportType, keyValueYarpcServer examplepb.KeyValueYarpcServer, sinkYarpcServer examplepb.SinkYarpcServer, - f func(examplepb.KeyValueYarpcClient, examplepb.SinkYarpcClient) error, + f func(*Clients) error, ) error { var procedures []transport.Procedure if keyValueYarpcServer != nil { @@ -57,14 +65,18 @@ func WithClients( if sinkYarpcServer != nil { procedures = append(procedures, examplepb.BuildSinkYarpcProcedures(sinkYarpcServer)...) } - return testutils.WithClientConfig( + return testutils.WithClientInfo( "example", procedures, transportType, - func(clientConfig transport.ClientConfig) error { + func(clientInfo *testutils.ClientInfo) error { return f( - examplepb.NewKeyValueYarpcClient(clientConfig), - examplepb.NewSinkYarpcClient(clientConfig), + &Clients{ + examplepb.NewKeyValueYarpcClient(clientInfo.ClientConfig), + examplepb.NewSinkYarpcClient(clientInfo.ClientConfig), + examplepb.NewKeyValueClient(clientInfo.GRPCClientConn), + examplepb.NewSinkClient(clientInfo.GRPCClientConn), + }, ) }, ) diff --git a/internal/examples/protobuf/main.go b/internal/examples/protobuf/main.go index eeeabe8d2..5aae41b50 100644 --- a/internal/examples/protobuf/main.go +++ b/internal/examples/protobuf/main.go @@ -49,17 +49,16 @@ func do() error { testutils.TransportTypeTChannel, keyValueYarpcServer, sinkYarpcServer, - func(keyValueYarpcClient examplepb.KeyValueYarpcClient, sinkYarpcClient examplepb.SinkYarpcClient) error { - return doClient(keyValueYarpcClient, sinkYarpcClient, keyValueYarpcServer, sinkYarpcServer) + func(clients *example.Clients) error { + return doClient(keyValueYarpcServer, sinkYarpcServer, clients) }, ) } func doClient( - keyValueYarpcClient examplepb.KeyValueYarpcClient, - sinkYarpcClient examplepb.SinkYarpcClient, keyValueYarpcServer *example.KeyValueYarpcServer, sinkYarpcServer *example.SinkYarpcServer, + clients *example.Clients, ) error { scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { @@ -80,7 +79,7 @@ func doClient( key := args[0] ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if response, err := keyValueYarpcClient.GetValue(ctx, &examplepb.GetValueRequest{key}); err != nil { + if response, err := clients.KeyValueYarpcClient.GetValue(ctx, &examplepb.GetValueRequest{key}); err != nil { fmt.Printf("get %s failed: %s\n", key, err.Error()) } else { fmt.Println(key, "=", response.Value) @@ -98,7 +97,7 @@ func doClient( } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if _, err := keyValueYarpcClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}); err != nil { + if _, err := clients.KeyValueYarpcClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}); err != nil { fmt.Printf("set %s = %s failed: %v\n", key, value, err.Error()) } continue @@ -110,7 +109,7 @@ func doClient( value := args[0] ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if _, err := sinkYarpcClient.Fire(ctx, &examplepb.FireRequest{value}); err != nil { + if _, err := clients.SinkYarpcClient.Fire(ctx, &examplepb.FireRequest{value}); err != nil { fmt.Printf("fire %s failed: %s\n", value, err.Error()) } if err := sinkYarpcServer.WaitFireDone(); err != nil { diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index fa6247c46..a8f33ac32 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -31,6 +31,8 @@ import ( "go.uber.org/yarpc/transport/http" "go.uber.org/yarpc/transport/tchannel" "go.uber.org/yarpc/transport/x/grpc" + + ggrpc "google.golang.org/grpc" ) const ( @@ -82,13 +84,19 @@ func ParseTransportType(s string) (TransportType, error) { } } +// ClientInfo holds the client info for testing. +type ClientInfo struct { + ClientConfig transport.ClientConfig + GRPCClientConn *ggrpc.ClientConn +} + // WithClientConfig wraps a function by setting up a client and server dispatcher and giving // the function the client configuration to use in tests for the given TransportType. // // The server dispatcher will be brought up using all TransportTypes and with the serviceName. // The client dispatcher will be brought up using the given TransportType for Unary, HTTP for // Oneway, and the serviceName with a "-client" suffix. -func WithClientConfig(serviceName string, procedures []transport.Procedure, transportType TransportType, f func(transport.ClientConfig) error) (err error) { +func WithClientInfo(serviceName string, procedures []transport.Procedure, transportType TransportType, f func(*ClientInfo) error) (err error) { dispatcherConfig, err := NewDispatcherConfig(serviceName) if err != nil { return err @@ -109,7 +117,20 @@ func WithClientConfig(serviceName string, procedures []transport.Procedure, tran return err } defer func() { err = errors.CombineErrors(err, clientDispatcher.Stop()) }() - return f(clientDispatcher.ClientConfig(serviceName)) + grpcPort, err := dispatcherConfig.GetPort(TransportTypeGRPC) + if err != nil { + return err + } + grpcClientConn, err := ggrpc.Dial(fmt.Sprintf("127.0.0.1:%d", grpcPort), ggrpc.WithInsecure()) + if err != nil { + return err + } + return f( + &ClientInfo{ + clientDispatcher.ClientConfig(serviceName), + grpcClientConn, + }, + ) } // NewClientDispatcher returns a new client Dispatcher. From 47cefc52a295d0a3c9f7261b41a850eea0daba4d Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 12:11:14 +0200 Subject: [PATCH 15/82] Update protobuf example input --- internal/examples/protobuf/test-expected-output.txt | 12 ++---------- internal/examples/protobuf/test-input.txt | 6 +----- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/internal/examples/protobuf/test-expected-output.txt b/internal/examples/protobuf/test-expected-output.txt index 4a8424184..c7a4363c9 100644 --- a/internal/examples/protobuf/test-expected-output.txt +++ b/internal/examples/protobuf/test-expected-output.txt @@ -1,19 +1,11 @@ -get foo -get foo failed: key not set: foo -get foo -get foo failed: key not set: foo set foo bar get foo foo = bar -get foo -foo = bar set baz qux get baz baz = qux -get foo -foo = bar -get baz -baz = qux +get bat +get bat failed: key not set: bat fire foo fire bar fired-values diff --git a/internal/examples/protobuf/test-input.txt b/internal/examples/protobuf/test-input.txt index d78fb83fa..0a522ff82 100644 --- a/internal/examples/protobuf/test-input.txt +++ b/internal/examples/protobuf/test-input.txt @@ -1,12 +1,8 @@ -get foo -get foo set foo bar get foo -get foo set baz qux get baz -get foo -get baz +get bat fire foo fire bar fired-values From d3937f386ea6e7021194148286d01fe49704b8c4 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 12:13:36 +0200 Subject: [PATCH 16/82] Fix --- internal/examples/protobuf/example/example.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/examples/protobuf/example/example.go b/internal/examples/protobuf/example/example.go index e70dd3918..f13b10ec6 100644 --- a/internal/examples/protobuf/example/example.go +++ b/internal/examples/protobuf/example/example.go @@ -47,7 +47,7 @@ var ( type Clients struct { KeyValueYarpcClient examplepb.KeyValueYarpcClient SinkYarpcClient examplepb.SinkYarpcClient - KeyGRPCClient examplepb.KeyValueClient + KeyValueGRPCClient examplepb.KeyValueClient SinkGRPCClient examplepb.SinkClient } From 54735d082730e9b12d407648d4bd2dafce9c83d6 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 12:20:37 +0200 Subject: [PATCH 17/82] FQSN --- encoding/x/protobuf/protoc-gen-yarpc-go/main.go | 4 ++-- .../examples/protobuf/examplepb/example.pb.yarpc.go | 8 ++++---- internal/protoplugin/protoplugin.go | 10 ++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/encoding/x/protobuf/protoc-gen-yarpc-go/main.go b/encoding/x/protobuf/protoc-gen-yarpc-go/main.go index 56064c7c9..91447dd5c 100644 --- a/encoding/x/protobuf/protoc-gen-yarpc-go/main.go +++ b/encoding/x/protobuf/protoc-gen-yarpc-go/main.go @@ -79,7 +79,7 @@ type {{$service.GetName}}YarpcClient interface { // New{{$service.GetName}}YarpcClient builds a new yarpc client for the {{$service.GetName}} service. func New{{$service.GetName}}YarpcClient(clientConfig transport.ClientConfig) {{$service.GetName}}YarpcClient { - return &_{{$service.GetName}}YarpcCaller{protobuf.NewClient("{{$service.GetName}}", clientConfig)} + return &_{{$service.GetName}}YarpcCaller{protobuf.NewClient("{{$service.FQSN}}", clientConfig)} } // {{$service.GetName}}YarpcServer is the yarpc server-side interface for the {{$service.GetName}} service. @@ -94,7 +94,7 @@ type {{$service.GetName}}YarpcServer interface { func Build{{$service.GetName}}YarpcProcedures(server {{$service.GetName}}YarpcServer) []transport.Procedure { handler := &_{{$service.GetName}}YarpcHandler{server} return protobuf.BuildProcedures( - "{{$service.GetName}}", + "{{$service.FQSN}}", map[string]transport.UnaryHandler{ {{range $method := unaryMethods $service}}"{{$method.GetName}}": protobuf.NewUnaryHandler(handler.{{$method.GetName}}, new{{$service.GetName}}_{{$method.GetName}}YarpcRequest), {{end}} diff --git a/internal/examples/protobuf/examplepb/example.pb.yarpc.go b/internal/examples/protobuf/examplepb/example.pb.yarpc.go index 27045c054..3881e9ea2 100644 --- a/internal/examples/protobuf/examplepb/example.pb.yarpc.go +++ b/internal/examples/protobuf/examplepb/example.pb.yarpc.go @@ -42,7 +42,7 @@ type KeyValueYarpcClient interface { // NewKeyValueYarpcClient builds a new yarpc client for the KeyValue service. func NewKeyValueYarpcClient(clientConfig transport.ClientConfig) KeyValueYarpcClient { - return &_KeyValueYarpcCaller{protobuf.NewClient("KeyValue", clientConfig)} + return &_KeyValueYarpcCaller{protobuf.NewClient(".uber.yarpc.internal.examples.protobuf.example.KeyValue", clientConfig)} } // KeyValueYarpcServer is the yarpc server-side interface for the KeyValue service. @@ -55,7 +55,7 @@ type KeyValueYarpcServer interface { func BuildKeyValueYarpcProcedures(server KeyValueYarpcServer) []transport.Procedure { handler := &_KeyValueYarpcHandler{server} return protobuf.BuildProcedures( - "KeyValue", + ".uber.yarpc.internal.examples.protobuf.example.KeyValue", map[string]transport.UnaryHandler{ "GetValue": protobuf.NewUnaryHandler(handler.GetValue, newKeyValue_GetValueYarpcRequest), "SetValue": protobuf.NewUnaryHandler(handler.SetValue, newKeyValue_SetValueYarpcRequest), @@ -158,7 +158,7 @@ type SinkYarpcClient interface { // NewSinkYarpcClient builds a new yarpc client for the Sink service. func NewSinkYarpcClient(clientConfig transport.ClientConfig) SinkYarpcClient { - return &_SinkYarpcCaller{protobuf.NewClient("Sink", clientConfig)} + return &_SinkYarpcCaller{protobuf.NewClient(".uber.yarpc.internal.examples.protobuf.example.Sink", clientConfig)} } // SinkYarpcServer is the yarpc server-side interface for the Sink service. @@ -170,7 +170,7 @@ type SinkYarpcServer interface { func BuildSinkYarpcProcedures(server SinkYarpcServer) []transport.Procedure { handler := &_SinkYarpcHandler{server} return protobuf.BuildProcedures( - "Sink", + ".uber.yarpc.internal.examples.protobuf.example.Sink", map[string]transport.UnaryHandler{}, map[string]transport.OnewayHandler{ "Fire": protobuf.NewOnewayHandler(handler.Fire, newSink_FireYarpcRequest), diff --git a/internal/protoplugin/protoplugin.go b/internal/protoplugin/protoplugin.go index e4f904dac..fb2c5f9bf 100644 --- a/internal/protoplugin/protoplugin.go +++ b/internal/protoplugin/protoplugin.go @@ -159,6 +159,16 @@ type Service struct { Methods []*Method } +// FQSN returns a fully qualified service name of this service. +func (s *Service) FQSN() string { + components := []string{""} + if s.File.Package != nil { + components = append(components, s.File.GetPackage()) + } + components = append(components, s.GetName()) + return strings.Join(components, ".") +} + // Method wraps descriptor.MethodDescriptorProto for richer features. type Method struct { *descriptor.MethodDescriptorProto From d1803f7004ea0cad007a323e10661c4ddcf531cf Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 12:24:25 +0200 Subject: [PATCH 18/82] fix --- encoding/x/protobuf/protoc-gen-yarpc-go/main.go | 11 ++++++++--- .../examples/protobuf/examplepb/example.pb.yarpc.go | 8 ++++---- internal/examples/protobuf/main.go | 4 ++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/encoding/x/protobuf/protoc-gen-yarpc-go/main.go b/encoding/x/protobuf/protoc-gen-yarpc-go/main.go index 91447dd5c..f46e8c035 100644 --- a/encoding/x/protobuf/protoc-gen-yarpc-go/main.go +++ b/encoding/x/protobuf/protoc-gen-yarpc-go/main.go @@ -50,6 +50,7 @@ package main import ( "fmt" "log" + "strings" "text/template" "go.uber.org/yarpc/internal/protoplugin" @@ -79,7 +80,7 @@ type {{$service.GetName}}YarpcClient interface { // New{{$service.GetName}}YarpcClient builds a new yarpc client for the {{$service.GetName}} service. func New{{$service.GetName}}YarpcClient(clientConfig transport.ClientConfig) {{$service.GetName}}YarpcClient { - return &_{{$service.GetName}}YarpcCaller{protobuf.NewClient("{{$service.FQSN}}", clientConfig)} + return &_{{$service.GetName}}YarpcCaller{protobuf.NewClient("{{trimPrefixPeriod $service.FQSN}}", clientConfig)} } // {{$service.GetName}}YarpcServer is the yarpc server-side interface for the {{$service.GetName}} service. @@ -94,7 +95,7 @@ type {{$service.GetName}}YarpcServer interface { func Build{{$service.GetName}}YarpcProcedures(server {{$service.GetName}}YarpcServer) []transport.Procedure { handler := &_{{$service.GetName}}YarpcHandler{server} return protobuf.BuildProcedures( - "{{$service.FQSN}}", + "{{trimPrefixPeriod $service.FQSN}}", map[string]transport.UnaryHandler{ {{range $method := unaryMethods $service}}"{{$method.GetName}}": protobuf.NewUnaryHandler(handler.{{$method.GetName}}, new{{$service.GetName}}_{{$method.GetName}}YarpcRequest), {{end}} @@ -181,7 +182,7 @@ var ( {{end}} ` -var funcMap = template.FuncMap{"unaryMethods": unaryMethods, "onewayMethods": onewayMethods} +var funcMap = template.FuncMap{"unaryMethods": unaryMethods, "onewayMethods": onewayMethods, "trimPrefixPeriod": trimPrefixPeriod} func main() { if err := protoplugin.Run( @@ -230,3 +231,7 @@ func onewayMethods(service *protoplugin.Service) ([]*protoplugin.Method, error) } return methods, nil } + +func trimPrefixPeriod(s string) string { + return strings.TrimPrefix(s, ".") +} diff --git a/internal/examples/protobuf/examplepb/example.pb.yarpc.go b/internal/examples/protobuf/examplepb/example.pb.yarpc.go index 3881e9ea2..2770fb0ac 100644 --- a/internal/examples/protobuf/examplepb/example.pb.yarpc.go +++ b/internal/examples/protobuf/examplepb/example.pb.yarpc.go @@ -42,7 +42,7 @@ type KeyValueYarpcClient interface { // NewKeyValueYarpcClient builds a new yarpc client for the KeyValue service. func NewKeyValueYarpcClient(clientConfig transport.ClientConfig) KeyValueYarpcClient { - return &_KeyValueYarpcCaller{protobuf.NewClient(".uber.yarpc.internal.examples.protobuf.example.KeyValue", clientConfig)} + return &_KeyValueYarpcCaller{protobuf.NewClient("uber.yarpc.internal.examples.protobuf.example.KeyValue", clientConfig)} } // KeyValueYarpcServer is the yarpc server-side interface for the KeyValue service. @@ -55,7 +55,7 @@ type KeyValueYarpcServer interface { func BuildKeyValueYarpcProcedures(server KeyValueYarpcServer) []transport.Procedure { handler := &_KeyValueYarpcHandler{server} return protobuf.BuildProcedures( - ".uber.yarpc.internal.examples.protobuf.example.KeyValue", + "uber.yarpc.internal.examples.protobuf.example.KeyValue", map[string]transport.UnaryHandler{ "GetValue": protobuf.NewUnaryHandler(handler.GetValue, newKeyValue_GetValueYarpcRequest), "SetValue": protobuf.NewUnaryHandler(handler.SetValue, newKeyValue_SetValueYarpcRequest), @@ -158,7 +158,7 @@ type SinkYarpcClient interface { // NewSinkYarpcClient builds a new yarpc client for the Sink service. func NewSinkYarpcClient(clientConfig transport.ClientConfig) SinkYarpcClient { - return &_SinkYarpcCaller{protobuf.NewClient(".uber.yarpc.internal.examples.protobuf.example.Sink", clientConfig)} + return &_SinkYarpcCaller{protobuf.NewClient("uber.yarpc.internal.examples.protobuf.example.Sink", clientConfig)} } // SinkYarpcServer is the yarpc server-side interface for the Sink service. @@ -170,7 +170,7 @@ type SinkYarpcServer interface { func BuildSinkYarpcProcedures(server SinkYarpcServer) []transport.Procedure { handler := &_SinkYarpcHandler{server} return protobuf.BuildProcedures( - ".uber.yarpc.internal.examples.protobuf.example.Sink", + "uber.yarpc.internal.examples.protobuf.example.Sink", map[string]transport.UnaryHandler{}, map[string]transport.OnewayHandler{ "Fire": protobuf.NewOnewayHandler(handler.Fire, newSink_FireYarpcRequest), diff --git a/internal/examples/protobuf/main.go b/internal/examples/protobuf/main.go index 5aae41b50..d0995dc94 100644 --- a/internal/examples/protobuf/main.go +++ b/internal/examples/protobuf/main.go @@ -79,7 +79,7 @@ func doClient( key := args[0] ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if response, err := clients.KeyValueYarpcClient.GetValue(ctx, &examplepb.GetValueRequest{key}); err != nil { + if response, err := clients.KeyValueGRPCClient.GetValue(ctx, &examplepb.GetValueRequest{key}); err != nil { fmt.Printf("get %s failed: %s\n", key, err.Error()) } else { fmt.Println(key, "=", response.Value) @@ -97,7 +97,7 @@ func doClient( } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if _, err := clients.KeyValueYarpcClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}); err != nil { + if _, err := clients.KeyValueGRPCClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}); err != nil { fmt.Printf("set %s = %s failed: %v\n", key, value, err.Error()) } continue From 3e38dfe199064f0f0a1d441be490bfda63fc603d Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 12:25:51 +0200 Subject: [PATCH 19/82] fix --- internal/examples/protobuf/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/examples/protobuf/main.go b/internal/examples/protobuf/main.go index d0995dc94..5aae41b50 100644 --- a/internal/examples/protobuf/main.go +++ b/internal/examples/protobuf/main.go @@ -79,7 +79,7 @@ func doClient( key := args[0] ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if response, err := clients.KeyValueGRPCClient.GetValue(ctx, &examplepb.GetValueRequest{key}); err != nil { + if response, err := clients.KeyValueYarpcClient.GetValue(ctx, &examplepb.GetValueRequest{key}); err != nil { fmt.Printf("get %s failed: %s\n", key, err.Error()) } else { fmt.Println(key, "=", response.Value) @@ -97,7 +97,7 @@ func doClient( } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if _, err := clients.KeyValueGRPCClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}); err != nil { + if _, err := clients.KeyValueYarpcClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}); err != nil { fmt.Printf("set %s = %s failed: %v\n", key, value, err.Error()) } continue From 45bf89c3f0de3b5ce9c84f949dbac57a8a45d018 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 12:30:49 +0200 Subject: [PATCH 20/82] Allow switching example to grpc client --- internal/examples/protobuf/main.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/examples/protobuf/main.go b/internal/examples/protobuf/main.go index 5aae41b50..e1c47d5c5 100644 --- a/internal/examples/protobuf/main.go +++ b/internal/examples/protobuf/main.go @@ -79,7 +79,14 @@ func doClient( key := args[0] ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if response, err := clients.KeyValueYarpcClient.GetValue(ctx, &examplepb.GetValueRequest{key}); err != nil { + var response *examplepb.GetValueResponse + var err error + if os.Getenv("EXAMPLE_GRPC") != "" { + response, err = clients.KeyValueGRPCClient.GetValue(ctx, &examplepb.GetValueRequest{key}) + } else { + response, err = clients.KeyValueYarpcClient.GetValue(ctx, &examplepb.GetValueRequest{key}) + } + if err != nil { fmt.Printf("get %s failed: %s\n", key, err.Error()) } else { fmt.Println(key, "=", response.Value) @@ -97,7 +104,13 @@ func doClient( } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if _, err := clients.KeyValueYarpcClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}); err != nil { + var err error + if os.Getenv("EXAMPLE_GRPC") != "" { + _, err = clients.KeyValueGRPCClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}) + } else { + _, err = clients.KeyValueYarpcClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}) + } + if err != nil { fmt.Printf("set %s = %s failed: %v\n", key, value, err.Error()) } continue From bec7415e1e3f61aabc8455b35c42421f1a910a9e Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 12:33:31 +0200 Subject: [PATCH 21/82] more logging --- transport/x/grpc/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index 91c92286a..cf4fc38fb 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -49,6 +49,6 @@ func (m *methodHandler) handle( if err := decodeFunc(&data); err != nil { return nil, err } - fmt.Printf("%s\n%s\n%+v\n%s\n", m.serviceName, m.methodName, ctx, string(data)) + fmt.Printf("%s\n%s\n%+v\n%TX%sX\n", m.serviceName, m.methodName, ctx, data, string(data)) return nil, nil } From 1589afa0246e539599b92e4282ca4fee80182d5d Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 13:10:18 +0200 Subject: [PATCH 22/82] More work --- transport/x/grpc/handlers.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index cf4fc38fb..4c5bc2777 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -21,6 +21,7 @@ package grpc import ( + "bytes" "fmt" "go.uber.org/yarpc/api/transport" @@ -52,3 +53,19 @@ func (m *methodHandler) handle( fmt.Printf("%s\n%s\n%+v\n%TX%sX\n", m.serviceName, m.methodName, ctx, data, string(data)) return nil, nil } + +func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func(interface{}) error) (*transport.Request, error) { + //md, ok := metadata.FromContext(ctx) + //if md == nil || !ok { + //return nil, fmt.Errorf("cannot get metadata from ctx: %v", ctx) + //} + var body []byte + if err := decodeFunc(&body); err != nil { + return nil, err + } + return &transport.Request{ + Service: m.serviceName, + Procedure: m.methodName, + Body: bytes.NewBuffer(body), + }, nil +} From 2b77918d1e80da46acdc71fb6dd2173e7d3aecd4 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 13:23:31 +0200 Subject: [PATCH 23/82] Transport writer --- transport/x/grpc/response_writer.go | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 transport/x/grpc/response_writer.go diff --git a/transport/x/grpc/response_writer.go b/transport/x/grpc/response_writer.go new file mode 100644 index 000000000..ecdb80d17 --- /dev/null +++ b/transport/x/grpc/response_writer.go @@ -0,0 +1,46 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "bytes" + "io" + + "go.uber.org/yarpc/api/transport" + + "google.golang.org/grpc/metadata" +) + +type responseWriter struct { + io.Writer + md metadata.MD +} + +func newResponseWriter() *responseWriter { + return &responseWriter{ + bytes.NewBuffer(nil), + make(metadata.MD, 0), + } +} + +func (r *responseWriter) AddHeaders(headers transport.Headers) {} + +func (r *responseWriter) SetAppllicationError() {} From 17d99d5ad65edd04f802fad2226d6519059fe4a6 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 15:11:10 +0200 Subject: [PATCH 24/82] Fix golint --- internal/testutils/testutils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index a8f33ac32..cc20b55e4 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -90,7 +90,7 @@ type ClientInfo struct { GRPCClientConn *ggrpc.ClientConn } -// WithClientConfig wraps a function by setting up a client and server dispatcher and giving +// WithClientInfo wraps a function by setting up a client and server dispatcher and giving // the function the client configuration to use in tests for the given TransportType. // // The server dispatcher will be brought up using all TransportTypes and with the serviceName. From d763d72dbf997cb304cbad972e374466ac75db95 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 16:57:37 +0200 Subject: [PATCH 25/82] Commit --- transport/x/grpc/handlers.go | 122 +++++++++++++++++++++++----- transport/x/grpc/headers.go | 116 ++++++++++++++++++++++++++ transport/x/grpc/inbound.go | 3 +- transport/x/grpc/response_writer.go | 7 +- 4 files changed, 224 insertions(+), 24 deletions(-) create mode 100644 transport/x/grpc/headers.go diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index 4c5bc2777..2f8dd40a2 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -23,21 +23,33 @@ package grpc import ( "bytes" "fmt" + "time" "go.uber.org/yarpc/api/transport" + "go.uber.org/yarpc/encoding/x/protobuf" + "go.uber.org/yarpc/internal/errors" + "go.uber.org/yarpc/internal/procedure" + "go.uber.org/yarpc/internal/request" "golang.org/x/net/context" "google.golang.org/grpc" + "google.golang.org/grpc/metadata" ) type methodHandler struct { - serviceName string - methodName string - router transport.Router + procedureServiceName string + serviceName string + methodName string + router transport.Router } -func newMethodHandler(serviceName string, methodName string, router transport.Router) *methodHandler { - return &methodHandler{serviceName, methodName, router} +func newMethodHandler( + procedureServiceName string, + serviceName string, + methodName string, + router transport.Router, +) *methodHandler { + return &methodHandler{procedureServiceName, serviceName, methodName, router} } func (m *methodHandler) handle( @@ -46,26 +58,98 @@ func (m *methodHandler) handle( decodeFunc func(interface{}) error, interceptor grpc.UnaryServerInterceptor, ) (interface{}, error) { + transportRequest, err := m.getTransportRequest(ctx, decodeFunc) + if err != nil { + return nil, err + } + //log.Printf("%+v\n", transportRequest) + if interceptor != nil { + return interceptor( + ctx, + transportRequest, + &grpc.UnaryServerInfo{ + noopGrpcStruct{}, + m.getFullMethod(), + }, + func(ctx context.Context, request interface{}) (interface{}, error) { + transportRequest, ok := request.(*transport.Request) + if !ok { + return nil, fmt.Errorf("expected *transport.Request, got %T", request) + } + return m.call(ctx, transportRequest) + }, + ) + } + return m.call(ctx, transportRequest) +} + +func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func(interface{}) error) (*transport.Request, error) { + md, ok := metadata.FromContext(ctx) + if md == nil || !ok { + return nil, fmt.Errorf("cannot get metadata from ctx: %v", ctx) + } + caller, err := getCaller(md) + if err != nil { + return nil, err + } + if caller == "" { + caller = m.serviceName + } + encoding, err := getEncoding(md) + if err != nil { + return nil, err + } + if encoding == "" { + encoding = protobuf.Encoding + } + headers, err := getApplicationHeaders(md) + if err != nil { + return nil, err + } var data []byte if err := decodeFunc(&data); err != nil { return nil, err } - fmt.Printf("%s\n%s\n%+v\n%TX%sX\n", m.serviceName, m.methodName, ctx, data, string(data)) - return nil, nil + transportRequest := &transport.Request{ + Caller: caller, + Encoding: encoding, + Service: m.procedureServiceName, + Procedure: procedure.ToName(m.serviceName, m.methodName), + Headers: headers, + Body: bytes.NewBuffer(data), + } + if err := transport.ValidateRequest(transportRequest); err != nil { + return nil, err + } + return transportRequest, nil } -func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func(interface{}) error) (*transport.Request, error) { - //md, ok := metadata.FromContext(ctx) - //if md == nil || !ok { - //return nil, fmt.Errorf("cannot get metadata from ctx: %v", ctx) - //} - var body []byte - if err := decodeFunc(&body); err != nil { +func (m *methodHandler) getFullMethod() string { + return fmt.Sprintf("/%s/%s", m.serviceName, m.methodName) +} + +func (m *methodHandler) call(ctx context.Context, transportRequest *transport.Request) (interface{}, error) { + handlerSpec, err := m.router.Choose(ctx, transportRequest) + if err != nil { + return nil, err + } + switch handlerSpec.Type() { + case transport.Unary: + return m.callUnary(ctx, transportRequest, handlerSpec.Unary()) + default: + return nil, errors.UnsupportedTypeError{"grpc", handlerSpec.Type().String()} + } +} + +func (m *methodHandler) callUnary(ctx context.Context, transportRequest *transport.Request, unaryHandler transport.UnaryHandler) (interface{}, error) { + if err := request.ValidateUnaryContext(ctx); err != nil { return nil, err } - return &transport.Request{ - Service: m.serviceName, - Procedure: m.methodName, - Body: bytes.NewBuffer(body), - }, nil + responseWriter := newResponseWriter() + // TODO: always return data? + err := transport.DispatchUnaryHandler(ctx, unaryHandler, time.Now(), transportRequest, responseWriter) + err = errors.CombineErrors(err, grpc.SendHeader(ctx, responseWriter.md)) + data := responseWriter.Bytes() + //log.Printf("%s %v\n", string(data), err) + return &data, err } diff --git a/transport/x/grpc/headers.go b/transport/x/grpc/headers.go new file mode 100644 index 000000000..30c3cb9c1 --- /dev/null +++ b/transport/x/grpc/headers.go @@ -0,0 +1,116 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "fmt" + "strings" + + "go.uber.org/yarpc/api/transport" + "google.golang.org/grpc/metadata" +) + +const ( + globalHeaderPrefix = "yarpc-" + reservedHeaderPrefix = globalHeaderPrefix + "reserved-" + applicationHeaderPrefix = globalHeaderPrefix + "app-" + callerHeader = reservedHeaderPrefix + "caller" + encodingHeader = reservedHeaderPrefix + "encoding" +) + +func addCaller(md metadata.MD, caller string) error { + return addToMetadata(md, callerHeader, caller) +} + +func addEncoding(md metadata.MD, encoding transport.Encoding) error { + return addToMetadata(md, encodingHeader, string(encoding)) +} + +// add headers into md as application headers +// return error if md already has a key defined that is defined in headers +func addApplicationHeaders(md metadata.MD, headers transport.Headers) error { + for key, value := range headers.Items() { + if err := addToMetadata(md, applicationHeaderPrefix+key, value); err != nil { + return err + } + } + return nil +} + +func getCaller(md metadata.MD) (string, error) { + return getFromMetadata(md, callerHeader) +} + +func getEncoding(md metadata.MD) (transport.Encoding, error) { + encoding, err := getFromMetadata(md, encodingHeader) + if err != nil { + return "", err + } + return transport.Encoding(encoding), nil +} + +// get application headers from md +// return error if any application error has more than one value +func getApplicationHeaders(md metadata.MD) (transport.Headers, error) { + headers := transport.NewHeadersWithCapacity(md.Len()) + for mdKey := range md { + key := strings.TrimPrefix(mdKey, applicationHeaderPrefix) + // not an application header + if key == mdKey { + continue + } + value, err := getFromMetadata(md, mdKey) + if err != nil { + return headers, err + } + headers.With(key, value) + } + return headers, nil +} + +// add to md +// return error if key already in md +func addToMetadata(md metadata.MD, key string, value string) error { + key = transport.CanonicalizeHeaderKey(key) + if _, ok := md[key]; ok { + return fmt.Errorf("duplicate key: %s", key) + } + md[key] = []string{value} + return nil +} + +// get from md +// return "" if not present +// return error if more than one value +func getFromMetadata(md metadata.MD, key string) (string, error) { + values, ok := md[key] + if !ok { + return "", nil + } + switch len(values) { + case 0: + return "", nil + case 1: + return values[0], nil + default: + return "", fmt.Errorf("key has more than one value: %s", key) + } +} diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 74fc12e2d..c75f8d426 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -167,7 +167,8 @@ func getServiceNameAndMethodDesc(router transport.Router, procedure transport.Pr methodName = url.QueryEscape(methodName) return serviceName, grpc.MethodDesc{ MethodName: methodName, - Handler: newMethodHandler(serviceName, methodName, router).handle, + // TODO: what if two procedures have the same serviceName and methodName, but a different service? + Handler: newMethodHandler(procedure.Service, serviceName, methodName, router).handle, }, nil } diff --git a/transport/x/grpc/response_writer.go b/transport/x/grpc/response_writer.go index ecdb80d17..f3edf1fbd 100644 --- a/transport/x/grpc/response_writer.go +++ b/transport/x/grpc/response_writer.go @@ -22,7 +22,6 @@ package grpc import ( "bytes" - "io" "go.uber.org/yarpc/api/transport" @@ -30,17 +29,17 @@ import ( ) type responseWriter struct { - io.Writer + *bytes.Buffer md metadata.MD } func newResponseWriter() *responseWriter { return &responseWriter{ bytes.NewBuffer(nil), - make(metadata.MD, 0), + metadata.New(nil), } } func (r *responseWriter) AddHeaders(headers transport.Headers) {} -func (r *responseWriter) SetAppllicationError() {} +func (r *responseWriter) SetApplicationError() {} From cc8c00a568598b2d244fb0d3b4cfc945a16b4384 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 17:32:54 +0200 Subject: [PATCH 26/82] Commit --- encoding/x/protobuf/inbound.go | 8 +++++++- encoding/x/protobuf/outbound.go | 9 +++++++++ encoding/x/protobuf/types.go | 16 ++++++++++++++-- transport/x/grpc/headers.go | 2 +- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/encoding/x/protobuf/inbound.go b/encoding/x/protobuf/inbound.go index ecadaf35d..a901af655 100644 --- a/encoding/x/protobuf/inbound.go +++ b/encoding/x/protobuf/inbound.go @@ -66,6 +66,9 @@ func (u *unaryHandler) Handle(ctx context.Context, transportRequest *transport.R } } response, appErr := u.handle(ctx, request) + if appErr != nil { + responseWriter.SetApplicationError() + } if err := call.WriteToResponse(responseWriter); err != nil { return err } @@ -78,9 +81,12 @@ func (u *unaryHandler) Handle(ctx context.Context, transportRequest *transport.R } responseData = protoBuffer.Bytes() } + if rawResponse, ok := ctx.Value(rawResponseKey).(bool); ok && rawResponse { + _, err := responseWriter.Write(responseData) + return err + } var wireError *wirepb.Error if appErr != nil { - responseWriter.SetApplicationError() wireError = &wirepb.Error{ appErr.Error(), } diff --git a/encoding/x/protobuf/outbound.go b/encoding/x/protobuf/outbound.go index 218c53d75..6198f9b74 100644 --- a/encoding/x/protobuf/outbound.go +++ b/encoding/x/protobuf/outbound.go @@ -78,6 +78,15 @@ func (c *client) Call( if responseData == nil { return nil, nil } + // TODO: the error from Call will be the application error, we might + // also have a response returned however + if rawResponse, ok := ctx.Value(rawResponseKey).(bool); ok && rawResponse { + response := newResponse() + if err := proto.Unmarshal(responseData, response); err != nil { + return nil, encoding.ResponseBodyDecodeError(transportRequest, err) + } + return response, nil + } wireResponse := &wirepb.Response{} if err := proto.Unmarshal(responseData, wireResponse); err != nil { return nil, encoding.ResponseBodyDecodeError(transportRequest, err) diff --git a/encoding/x/protobuf/types.go b/encoding/x/protobuf/types.go index 4fa53cbc5..6687976b0 100644 --- a/encoding/x/protobuf/types.go +++ b/encoding/x/protobuf/types.go @@ -31,8 +31,12 @@ import ( "github.com/gogo/protobuf/proto" ) -// Encoding is the name of this encoding. -const Encoding transport.Encoding = "protobuf" +const ( + // Encoding is the name of this encoding. + Encoding transport.Encoding = "protobuf" + + rawResponseKey = "yarpc-protobuf-raw-response" +) // GetApplicationError returns the application error from the server, if present. // @@ -44,6 +48,14 @@ func GetApplicationError(err error) error { return nil } +// WithRawResponse will return a new Context that signals a UnaryHandler +// to not encode an application error inside a wirepb.Response object, instead +// marshalling the actual response, and returning the application error as an +// error from Handle. +func WithRawResponse(ctx context.Context) context.Context { + return context.WithValue(ctx, rawResponseKey, true) +} + // ***all below functions should only be called by generated code*** // BuildProcedures builds the transport.Procedures. diff --git a/transport/x/grpc/headers.go b/transport/x/grpc/headers.go index 30c3cb9c1..c9f2a6468 100644 --- a/transport/x/grpc/headers.go +++ b/transport/x/grpc/headers.go @@ -29,7 +29,7 @@ import ( ) const ( - globalHeaderPrefix = "yarpc-" + globalHeaderPrefix = "yarpc-grpc-" reservedHeaderPrefix = globalHeaderPrefix + "reserved-" applicationHeaderPrefix = globalHeaderPrefix + "app-" callerHeader = reservedHeaderPrefix + "caller" From c96fc1d4af11f45a58667823548027050957031d Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 17:36:00 +0200 Subject: [PATCH 27/82] Commit --- transport/x/grpc/handlers.go | 1 + transport/x/grpc/response_writer.go | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index 2f8dd40a2..7f8dbc9a0 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -142,6 +142,7 @@ func (m *methodHandler) call(ctx context.Context, transportRequest *transport.Re } func (m *methodHandler) callUnary(ctx context.Context, transportRequest *transport.Request, unaryHandler transport.UnaryHandler) (interface{}, error) { + ctx = protobuf.WithRawResponse(ctx) if err := request.ValidateUnaryContext(ctx); err != nil { return nil, err } diff --git a/transport/x/grpc/response_writer.go b/transport/x/grpc/response_writer.go index f3edf1fbd..3a6f33663 100644 --- a/transport/x/grpc/response_writer.go +++ b/transport/x/grpc/response_writer.go @@ -30,16 +30,23 @@ import ( type responseWriter struct { *bytes.Buffer - md metadata.MD + md metadata.MD + isApplicationError bool } func newResponseWriter() *responseWriter { return &responseWriter{ bytes.NewBuffer(nil), metadata.New(nil), + false, } } -func (r *responseWriter) AddHeaders(headers transport.Headers) {} +func (r *responseWriter) AddHeaders(headers transport.Headers) { + // TODO: handle error + _ = addApplicationHeaders(r.md, headers) +} -func (r *responseWriter) SetApplicationError() {} +func (r *responseWriter) SetApplicationError() { + r.isApplicationError = true +} From e43151e3ebc4b101e29265324f2a3c8f71447062 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 29 Mar 2017 17:39:14 +0200 Subject: [PATCH 28/82] Working --- encoding/x/protobuf/inbound.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/encoding/x/protobuf/inbound.go b/encoding/x/protobuf/inbound.go index a901af655..6ee18580c 100644 --- a/encoding/x/protobuf/inbound.go +++ b/encoding/x/protobuf/inbound.go @@ -83,7 +83,10 @@ func (u *unaryHandler) Handle(ctx context.Context, transportRequest *transport.R } if rawResponse, ok := ctx.Value(rawResponseKey).(bool); ok && rawResponse { _, err := responseWriter.Write(responseData) - return err + if err != nil { + return err + } + return appErr } var wireError *wirepb.Error if appErr != nil { From ec85413f6735436b839d023a905b4c3a4edc6794 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 16:13:49 +0200 Subject: [PATCH 29/82] Commit --- transport/x/grpc/inbound.go | 13 ++----- transport/x/grpc/outbound.go | 68 +++++++++++++++++++++++++++++++++++- transport/x/grpc/util.go | 29 +++++++++++++++ 3 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 transport/x/grpc/util.go diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index c75f8d426..43d43c144 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -22,15 +22,12 @@ package grpc import ( "errors" - "fmt" "net" - "net/url" "sync" "google.golang.org/grpc" "go.uber.org/yarpc/api/transport" - internalprocedure "go.uber.org/yarpc/internal/procedure" internalsync "go.uber.org/yarpc/internal/sync" ) @@ -157,14 +154,10 @@ func getServiceDescs(router transport.Router) ([]*grpc.ServiceDesc, error) { } func getServiceNameAndMethodDesc(router transport.Router, procedure transport.Procedure) (string, grpc.MethodDesc, error) { - serviceName, methodName := internalprocedure.FromName(procedure.Name) - if serviceName == "" || methodName == "" { - return "", grpc.MethodDesc{}, fmt.Errorf("invalid procedure name: %s", procedure.Name) + serviceName, methodName, err := procedureNameToServiceNameMethodName(procedure.Name) + if err != nil { + return "", grpc.MethodDesc{}, err } - // TODO: do we really need to do url.QueryEscape? - // Are there consequences if there is a diff from the string and the url.QueryEscape string? - serviceName = url.QueryEscape(serviceName) - methodName = url.QueryEscape(methodName) return serviceName, grpc.MethodDesc{ MethodName: methodName, // TODO: what if two procedures have the same serviceName and methodName, but a different service? diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index c0feaad2c..83039132b 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -21,12 +21,18 @@ package grpc import ( + "bytes" "context" + "io/ioutil" "sync" + "time" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" "go.uber.org/yarpc/api/transport" + "go.uber.org/yarpc/internal/errors" internalsync "go.uber.org/yarpc/internal/sync" ) @@ -67,7 +73,35 @@ func (o *Outbound) Transports() []transport.Transport { // Call implements transport.UnaryOutbound#Call. func (o *Outbound) Call(ctx context.Context, request *transport.Request) (*transport.Response, error) { - return nil, nil + start := time.Now() + md, err := requestToMetadata(request) + if err != nil { + return nil, err + } + requestBody, err := ioutil.ReadAll(request.Body) + if err != nil { + return nil, err + } + var responseBody []byte + responseMD := metadata.New(nil) + if err := grpc.Invoke( + metadata.NewContext(ctx, md), + prodecureNameToFullMethod(request.Procedure), + &requestBody, + &responseBody, + o.clientConn, + grpc.Header(&responseMD), + ); err != nil { + return nil, errorToGRPCError(ctx, request, err) + } + responseHeaders, err := getApplicationHeaders(responseMD) + if err != nil { + return nil, err + } + return &transport.Response{ + Body: ioutil.NopCloser(bytes.NewBuffer(responseBody)), + Headers: responseHeaders, + }, nil } func (o *Outbound) start() error { @@ -95,3 +129,35 @@ func (o *Outbound) stop() error { } return nil } + +func requestToMetadata(request *transport.Request) (metadata.MD, error) { + md = metadata.New(nil) + if err := addCaller(md, request.Caller); err != nil { + return nil, err + } + if err := addEncoding(md, request.Encoding); err != nil { + return nil, err + } + if err := addApplicationHeaders(md, request.Headers); err != nil { + return nil, err + } + return md, nil +} + +func errorToGRPCError(ctx context.Context, request *transport.Request, start time.Time, err error) error { + deadline, _ := ctx.Deadline() + ttl := deadline.Sub(start) + switch grpc.Code(err) { + case codes.DeadlineExceeded: + return errors.ClientTimeoutError(request.Service, request.Procedure, ttl) + case codes.Unimplemented, codes.InvalidArgument, codes.NotFound: + return errors.RemoteBadRequestError(grpc.ErrorDesc(err)) + case codes.Canceled, codes.AlreadyExists, codes.PermissionDenied, + codes.Unauthenticated, codes.ResourceExhausted, codes.FailedPrecondition, + codes.Aborted, codes.OutOfRange, codes.Internal, + codes.Unavailable, codes.DataLoss, codes.Unknown: + fallthrough + default: + return errors.RemoteUnexpectedError(grpc.ErrorDesc(err)) + } +} diff --git a/transport/x/grpc/util.go b/transport/x/grpc/util.go new file mode 100644 index 000000000..66fa687e6 --- /dev/null +++ b/transport/x/grpc/util.go @@ -0,0 +1,29 @@ +package grpc + +import ( + "fmt" + + "go.uber.org/yarpc/internal/procedure" +) + +func procedureNameToServiceNameMethodName(procedureName string) (string, string, error) { + serviceName, methodName := procedure.FromName(procedureName) + if serviceName == "" || methodName == "" { + return "", "", fmt.Errorf("invalid procedure name: %s", procedureName) + } + // TODO: do we really need to do url.QueryEscape? + // Are there consequences if there is a diff from the string and the url.QueryEscape string? + return uri.QueryEscape(serviceName), uri.QueryEscape(methodName), nil +} + +func prodecureNameToFullMethod(procedureName string) (string, error) { + serviceName, methodName, err := procedureNameToServiceNameMethodName(procedureName) + if err != nil { + return "", err + } + return toFullMethod(serviceName, methodName), nil +} + +func toFullMethod(serviceName string, methodName string) string { + return fmt.Sprintf("/%s/%s", serviceName, methodName) +} From 1ebc702dbc0355f1a806052d8b666c9165c7479c Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 16:20:09 +0200 Subject: [PATCH 30/82] Fixes --- glide.lock | 4 ++-- transport/x/grpc/outbound.go | 10 +++++++--- transport/x/grpc/util.go | 3 ++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/glide.lock b/glide.lock index a81bdffc4..0520c7aa7 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: b7e66eaece8f1b654ec6fe309277d324cfc4fea3cf151000bf964c94b35d3ad7 -updated: 2017-03-29T18:53:41.426622335+02:00 +updated: 2017-03-30T16:19:54.163856022+02:00 imports: - name: github.com/apache/thrift version: b2a4d4ae21c789b689dd162deb819665567f481c @@ -178,7 +178,7 @@ imports: - zapcore - zaptest/observer - name: golang.org/x/net - version: 05d3205156cb1bdc096578d6457fa161cd950aa2 + version: ffcf1bedda3b04ebb15a168a59800a73d6dc0f4d subpackages: - context - context/ctxhttp diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 83039132b..7cd8f1641 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -82,17 +82,21 @@ func (o *Outbound) Call(ctx context.Context, request *transport.Request) (*trans if err != nil { return nil, err } + fullMethod, err := prodecureNameToFullMethod(request.Procedure) + if err != nil { + return nil, err + } var responseBody []byte responseMD := metadata.New(nil) if err := grpc.Invoke( metadata.NewContext(ctx, md), - prodecureNameToFullMethod(request.Procedure), + fullMethod, &requestBody, &responseBody, o.clientConn, grpc.Header(&responseMD), ); err != nil { - return nil, errorToGRPCError(ctx, request, err) + return nil, errorToGRPCError(ctx, request, start, err) } responseHeaders, err := getApplicationHeaders(responseMD) if err != nil { @@ -131,7 +135,7 @@ func (o *Outbound) stop() error { } func requestToMetadata(request *transport.Request) (metadata.MD, error) { - md = metadata.New(nil) + md := metadata.New(nil) if err := addCaller(md, request.Caller); err != nil { return nil, err } diff --git a/transport/x/grpc/util.go b/transport/x/grpc/util.go index 66fa687e6..7dd6f4932 100644 --- a/transport/x/grpc/util.go +++ b/transport/x/grpc/util.go @@ -2,6 +2,7 @@ package grpc import ( "fmt" + "net/url" "go.uber.org/yarpc/internal/procedure" ) @@ -13,7 +14,7 @@ func procedureNameToServiceNameMethodName(procedureName string) (string, string, } // TODO: do we really need to do url.QueryEscape? // Are there consequences if there is a diff from the string and the url.QueryEscape string? - return uri.QueryEscape(serviceName), uri.QueryEscape(methodName), nil + return url.QueryEscape(serviceName), url.QueryEscape(methodName), nil } func prodecureNameToFullMethod(procedureName string) (string, error) { From 87a413562b924f425638a2fb51028d21dedd2751 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 16:28:47 +0200 Subject: [PATCH 31/82] Options on protobuf example --- internal/examples/protobuf/Makefile | 2 +- internal/examples/protobuf/main.go | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/internal/examples/protobuf/Makefile b/internal/examples/protobuf/Makefile index 533bed8a0..83cb21020 100644 --- a/internal/examples/protobuf/Makefile +++ b/internal/examples/protobuf/Makefile @@ -3,7 +3,7 @@ all: test .PHONY: run run: - cat test-input.txt | go run main.go + cat test-input.txt | go run main.go $(FLAGS) .PHONY: test: diff --git a/internal/examples/protobuf/main.go b/internal/examples/protobuf/main.go index e1c47d5c5..149e183d3 100644 --- a/internal/examples/protobuf/main.go +++ b/internal/examples/protobuf/main.go @@ -23,6 +23,7 @@ package main import ( "bufio" "context" + "flag" "fmt" "log" "os" @@ -34,19 +35,25 @@ import ( "go.uber.org/yarpc/internal/testutils" ) +var flagOutbound = flag.String("outbound", "tchannel", "The outbound to use for unary calls") +var flagGoogleGRPC = flag.Bool("google-grpc", false, "Use google grpc for outbound KeyValue calls") + func main() { + flag.Parse() if err := do(); err != nil { log.Fatal(err) } } func do() error { + transportType, err := testutils.ParseTransportType(*flagOutbound) + if err != nil { + return err + } keyValueYarpcServer := example.NewKeyValueYarpcServer() sinkYarpcServer := example.NewSinkYarpcServer(true) - // TChannel will be used for unary - // HTTP wil be used for oneway return example.WithClients( - testutils.TransportTypeTChannel, + transportType, keyValueYarpcServer, sinkYarpcServer, func(clients *example.Clients) error { @@ -81,7 +88,7 @@ func doClient( defer cancel() var response *examplepb.GetValueResponse var err error - if os.Getenv("EXAMPLE_GRPC") != "" { + if *flagGoogleGRPC { response, err = clients.KeyValueGRPCClient.GetValue(ctx, &examplepb.GetValueRequest{key}) } else { response, err = clients.KeyValueYarpcClient.GetValue(ctx, &examplepb.GetValueRequest{key}) @@ -105,7 +112,7 @@ func doClient( ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() var err error - if os.Getenv("EXAMPLE_GRPC") != "" { + if *flagGoogleGRPC { _, err = clients.KeyValueGRPCClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}) } else { _, err = clients.KeyValueYarpcClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}) From 0ce6d0d6c61f49e74b973fe5f4ab6cd69a2de103 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 17:06:36 +0200 Subject: [PATCH 32/82] Tests working --- encoding/x/protobuf/inbound.go | 3 +- encoding/x/protobuf/outbound.go | 2 +- encoding/x/protobuf/testing/testing_test.go | 6 ---- encoding/x/protobuf/types.go | 32 ++++++++++----------- internal/examples/protobuf/Makefile | 4 +-- internal/examples/protobuf/main.go | 9 ++++++ internal/testutils/testutils.go | 2 +- transport/x/grpc/handlers.go | 2 +- 8 files changed, 32 insertions(+), 28 deletions(-) diff --git a/encoding/x/protobuf/inbound.go b/encoding/x/protobuf/inbound.go index 6ee18580c..5f4ae1f4c 100644 --- a/encoding/x/protobuf/inbound.go +++ b/encoding/x/protobuf/inbound.go @@ -81,7 +81,8 @@ func (u *unaryHandler) Handle(ctx context.Context, transportRequest *transport.R } responseData = protoBuffer.Bytes() } - if rawResponse, ok := ctx.Value(rawResponseKey).(bool); ok && rawResponse { + if isRawResponse(transportRequest.Headers) { + responseWriter.AddHeaders(getRawResponseHeaders()) _, err := responseWriter.Write(responseData) if err != nil { return err diff --git a/encoding/x/protobuf/outbound.go b/encoding/x/protobuf/outbound.go index 6198f9b74..92a24df7e 100644 --- a/encoding/x/protobuf/outbound.go +++ b/encoding/x/protobuf/outbound.go @@ -80,7 +80,7 @@ func (c *client) Call( } // TODO: the error from Call will be the application error, we might // also have a response returned however - if rawResponse, ok := ctx.Value(rawResponseKey).(bool); ok && rawResponse { + if isRawResponse(transportResponse.Headers) { response := newResponse() if err := proto.Unmarshal(responseData, response); err != nil { return nil, encoding.ResponseBodyDecodeError(transportRequest, err) diff --git a/encoding/x/protobuf/testing/testing_test.go b/encoding/x/protobuf/testing/testing_test.go index 94291da5a..cea4c6739 100644 --- a/encoding/x/protobuf/testing/testing_test.go +++ b/encoding/x/protobuf/testing/testing_test.go @@ -25,7 +25,6 @@ import ( "testing" "time" - "go.uber.org/yarpc/encoding/x/protobuf" "go.uber.org/yarpc/internal/examples/protobuf/example" "go.uber.org/yarpc/internal/examples/protobuf/examplepb" "go.uber.org/yarpc/internal/testutils" @@ -36,9 +35,6 @@ import ( func TestIntegration(t *testing.T) { t.Parallel() for _, transportType := range testutils.AllTransportTypes { - if transportType == testutils.TransportTypeGRPC { - continue - } transportType := transportType t.Run(transportType.String(), func(t *testing.T) { testIntegrationForTransportType(t, transportType) }) } @@ -70,7 +66,6 @@ func testIntegration( ) { _, err := getValue(keyValueYarpcClient, "foo") assert.Error(t, err) - assert.NotNil(t, protobuf.GetApplicationError(err)) assert.NoError(t, setValue(keyValueYarpcClient, "foo", "bar")) value, err := getValue(keyValueYarpcClient, "foo") @@ -80,7 +75,6 @@ func testIntegration( assert.NoError(t, setValue(keyValueYarpcClient, "foo", "")) _, err = getValue(keyValueYarpcClient, "foo") assert.Error(t, err) - assert.NotNil(t, protobuf.GetApplicationError(err)) assert.NoError(t, setValue(keyValueYarpcClient, "foo", "baz")) assert.NoError(t, setValue(keyValueYarpcClient, "baz", "bat")) diff --git a/encoding/x/protobuf/types.go b/encoding/x/protobuf/types.go index 6687976b0..132615a10 100644 --- a/encoding/x/protobuf/types.go +++ b/encoding/x/protobuf/types.go @@ -35,25 +35,16 @@ const ( // Encoding is the name of this encoding. Encoding transport.Encoding = "protobuf" - rawResponseKey = "yarpc-protobuf-raw-response" + rawResponseHeaderKey = "yarpc-protobuf-raw-response" ) -// GetApplicationError returns the application error from the server, if present. -// -// TODO: this has overlap with IsApplicationError -func GetApplicationError(err error) error { - if applicationError, ok := err.(*applicationError); ok { - return applicationError - } - return nil -} +// SetRawResponse will set rawResponseHeaderKey to "true". -// WithRawResponse will return a new Context that signals a UnaryHandler -// to not encode an application error inside a wirepb.Response object, instead -// marshalling the actual response, and returning the application error as an -// error from Handle. -func WithRawResponse(ctx context.Context) context.Context { - return context.WithValue(ctx, rawResponseKey, true) +// rawResponseHeaderKey is a header key attached to either a request or +// response that signals a UnaryHandler to not encode an application error +// inside a wirepb.Response object, instead marshalling the actual response. +func SetRawResponse(headers transport.Headers) transport.Headers { + return headers.With(rawResponseHeaderKey, "1") } // ***all below functions should only be called by generated code*** @@ -130,3 +121,12 @@ func NewOnewayHandler( func CastError(expectedType proto.Message, actualType proto.Message) error { return fmt.Errorf("expected proto.Message to have type %T but had type %T", expectedType, actualType) } + +func isRawResponse(headers transport.Headers) bool { + rawResponse, ok := headers.Get(rawResponseHeaderKey) + return ok && rawResponse == "1" +} + +func getRawResponseHeaders() transport.Headers { + return SetRawResponse(transport.Headers{}) +} diff --git a/internal/examples/protobuf/Makefile b/internal/examples/protobuf/Makefile index 83cb21020..32ca701ac 100644 --- a/internal/examples/protobuf/Makefile +++ b/internal/examples/protobuf/Makefile @@ -5,8 +5,8 @@ all: test run: cat test-input.txt | go run main.go $(FLAGS) -.PHONY: +.PHONY: test test: $(eval TEST_ACTUAL_OUTPUT_TXT := $(shell mktemp -t protobuf-output.XXXXX)) - @cat test-input.txt | go run main.go > $(TEST_ACTUAL_OUTPUT_TXT) + @cat test-input.txt | go run main.go $(FLAGS) > $(TEST_ACTUAL_OUTPUT_TXT) @diff test-expected-output.txt $(TEST_ACTUAL_OUTPUT_TXT) || echo "error: diff from expected output" diff --git a/internal/examples/protobuf/main.go b/internal/examples/protobuf/main.go index 149e183d3..5402bc0aa 100644 --- a/internal/examples/protobuf/main.go +++ b/internal/examples/protobuf/main.go @@ -23,6 +23,7 @@ package main import ( "bufio" "context" + "errors" "flag" "fmt" "log" @@ -30,6 +31,8 @@ import ( "strings" "time" + "google.golang.org/grpc" + "go.uber.org/yarpc/internal/examples/protobuf/example" "go.uber.org/yarpc/internal/examples/protobuf/examplepb" "go.uber.org/yarpc/internal/testutils" @@ -90,6 +93,9 @@ func doClient( var err error if *flagGoogleGRPC { response, err = clients.KeyValueGRPCClient.GetValue(ctx, &examplepb.GetValueRequest{key}) + if err != nil { + err = errors.New(grpc.ErrorDesc(err)) + } } else { response, err = clients.KeyValueYarpcClient.GetValue(ctx, &examplepb.GetValueRequest{key}) } @@ -114,6 +120,9 @@ func doClient( var err error if *flagGoogleGRPC { _, err = clients.KeyValueGRPCClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}) + if err != nil { + err = errors.New(grpc.ErrorDesc(err)) + } } else { _, err = clients.KeyValueYarpcClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}) } diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index cc20b55e4..7a0534e1d 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -161,7 +161,7 @@ func NewClientDispatcher(transportType TransportType, config *DispatcherConfig) unaryOutbound = httpOutbound case TransportTypeGRPC: onewayOutbound = http.NewTransport().NewSingleOutbound(fmt.Sprintf("http://127.0.0.1:%d", httpPort)) - unaryOutbound = grpc.NewSingleOutbound(fmt.Sprintf("http://127.0.0.1:%d", port)) + unaryOutbound = grpc.NewSingleOutbound(fmt.Sprintf("127.0.0.1:%d", port)) default: return nil, fmt.Errorf("invalid TransportType: %v", transportType) } diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index 7f8dbc9a0..abbf00886 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -142,10 +142,10 @@ func (m *methodHandler) call(ctx context.Context, transportRequest *transport.Re } func (m *methodHandler) callUnary(ctx context.Context, transportRequest *transport.Request, unaryHandler transport.UnaryHandler) (interface{}, error) { - ctx = protobuf.WithRawResponse(ctx) if err := request.ValidateUnaryContext(ctx); err != nil { return nil, err } + protobuf.SetRawResponse(transportRequest.Headers) responseWriter := newResponseWriter() // TODO: always return data? err := transport.DispatchUnaryHandler(ctx, unaryHandler, time.Now(), transportRequest, responseWriter) From 7fb63ef525bdbdbd39d5af9a6ca075ca7c972f5d Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 17:06:46 +0200 Subject: [PATCH 33/82] Tests working --- encoding/x/protobuf/testing/benchmark_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/encoding/x/protobuf/testing/benchmark_test.go b/encoding/x/protobuf/testing/benchmark_test.go index 956be882b..31b632f65 100644 --- a/encoding/x/protobuf/testing/benchmark_test.go +++ b/encoding/x/protobuf/testing/benchmark_test.go @@ -30,9 +30,6 @@ import ( func BenchmarkIntegration(b *testing.B) { for _, transportType := range testutils.AllTransportTypes { - if transportType == testutils.TransportTypeGRPC { - continue - } b.Run(transportType.String(), func(b *testing.B) { benchmarkIntegrationForTransportType(b, transportType) }) } } From 9e8fc53ebe08ad00bdf132882231441817b6d97d Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 17:11:00 +0200 Subject: [PATCH 34/82] More testing --- encoding/x/protobuf/testing/testing_test.go | 51 +++++++++++++++------ 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/encoding/x/protobuf/testing/testing_test.go b/encoding/x/protobuf/testing/testing_test.go index cea4c6739..cc9da37f3 100644 --- a/encoding/x/protobuf/testing/testing_test.go +++ b/encoding/x/protobuf/testing/testing_test.go @@ -50,7 +50,7 @@ func testIntegrationForTransportType(t *testing.T, transportType testutils.Trans keyValueYarpcServer, sinkYarpcServer, func(clients *example.Clients) error { - testIntegration(t, clients.KeyValueYarpcClient, clients.SinkYarpcClient, keyValueYarpcServer, sinkYarpcServer) + testIntegration(t, clients, keyValueYarpcServer, sinkYarpcServer) return nil }, ), @@ -59,35 +59,41 @@ func testIntegrationForTransportType(t *testing.T, transportType testutils.Trans func testIntegration( t *testing.T, - keyValueYarpcClient examplepb.KeyValueYarpcClient, - sinkYarpcClient examplepb.SinkYarpcClient, + clients *example.Clients, keyValueYarpcServer *example.KeyValueYarpcServer, sinkYarpcServer *example.SinkYarpcServer, ) { - _, err := getValue(keyValueYarpcClient, "foo") + _, err := getValue(clients.KeyValueYarpcClient, "foo") + assert.Error(t, err) + _, err = getValueGRPC(clients.KeyValueGRPCClient, "foo") assert.Error(t, err) - assert.NoError(t, setValue(keyValueYarpcClient, "foo", "bar")) - value, err := getValue(keyValueYarpcClient, "foo") + assert.NoError(t, setValue(clients.KeyValueYarpcClient, "foo", "bar")) + value, err := getValue(clients.KeyValueYarpcClient, "foo") assert.NoError(t, err) assert.Equal(t, "bar", value) - assert.NoError(t, setValue(keyValueYarpcClient, "foo", "")) - _, err = getValue(keyValueYarpcClient, "foo") + assert.NoError(t, setValueGRPC(clients.KeyValueGRPCClient, "foo", "barGRPC")) + value, err = getValueGRPC(clients.KeyValueGRPCClient, "foo") + assert.NoError(t, err) + assert.Equal(t, "barGRPC", value) + + assert.NoError(t, setValue(clients.KeyValueYarpcClient, "foo", "")) + _, err = getValue(clients.KeyValueYarpcClient, "foo") assert.Error(t, err) - assert.NoError(t, setValue(keyValueYarpcClient, "foo", "baz")) - assert.NoError(t, setValue(keyValueYarpcClient, "baz", "bat")) - value, err = getValue(keyValueYarpcClient, "foo") + assert.NoError(t, setValue(clients.KeyValueYarpcClient, "foo", "baz")) + assert.NoError(t, setValue(clients.KeyValueYarpcClient, "baz", "bat")) + value, err = getValue(clients.KeyValueYarpcClient, "foo") assert.NoError(t, err) assert.Equal(t, "baz", value) - value, err = getValue(keyValueYarpcClient, "baz") + value, err = getValue(clients.KeyValueYarpcClient, "baz") assert.NoError(t, err) assert.Equal(t, "bat", value) - assert.NoError(t, fire(sinkYarpcClient, "foo")) + assert.NoError(t, fire(clients.SinkYarpcClient, "foo")) assert.NoError(t, sinkYarpcServer.WaitFireDone()) - assert.NoError(t, fire(sinkYarpcClient, "bar")) + assert.NoError(t, fire(clients.SinkYarpcClient, "bar")) assert.NoError(t, sinkYarpcServer.WaitFireDone()) assert.Equal(t, []string{"foo", "bar"}, sinkYarpcServer.Values()) } @@ -109,6 +115,23 @@ func setValue(keyValueYarpcClient examplepb.KeyValueYarpcClient, key string, val return err } +func getValueGRPC(keyValueGRPCClient examplepb.KeyValueClient, key string) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + response, err := keyValueGRPCClient.GetValue(ctx, &examplepb.GetValueRequest{key}) + if err != nil { + return "", err + } + return response.Value, nil +} + +func setValueGRPC(keyValueGRPCClient examplepb.KeyValueClient, key string, value string) error { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + _, err := keyValueGRPCClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}) + return err +} + func fire(sinkYarpcClient examplepb.SinkYarpcClient, value string) error { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() From 1dec7c5bfd24f5ced89cf6c47f997408e697bc07 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 17:21:30 +0200 Subject: [PATCH 35/82] Make yarpcmeta procedures work with grpc --- x/yarpcmeta/service.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x/yarpcmeta/service.go b/x/yarpcmeta/service.go index 7dc9fc318..05075c70f 100644 --- a/x/yarpcmeta/service.go +++ b/x/yarpcmeta/service.go @@ -66,9 +66,9 @@ func (m *service) Procedures() []transport.Procedure { Handler interface{} Signature string }{ - {"yarpc/procedures", m.procs, + {"yarpc::procedures", m.procs, `procedures() {"service": "...", "procedures": [{"name": "..."}]}`}, - {"yarpc/introspect", m.introspect, + {"yarpc::introspect", m.introspect, `introspect() {...}`}, } var r []transport.Procedure From d223951df77f26cdddc6f423900cd986e05e41a5 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 17:30:13 +0200 Subject: [PATCH 36/82] Make generate --- transport/x/grpc/util.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/transport/x/grpc/util.go b/transport/x/grpc/util.go index 7dd6f4932..e04c4007c 100644 --- a/transport/x/grpc/util.go +++ b/transport/x/grpc/util.go @@ -1,3 +1,23 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package grpc import ( From ab09c972f5e962d4df827eb9b0f12474edf73fc4 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 17:37:35 +0200 Subject: [PATCH 37/82] Fix lint --- encoding/x/protobuf/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/encoding/x/protobuf/types.go b/encoding/x/protobuf/types.go index 132615a10..5985362df 100644 --- a/encoding/x/protobuf/types.go +++ b/encoding/x/protobuf/types.go @@ -39,7 +39,7 @@ const ( ) // SetRawResponse will set rawResponseHeaderKey to "true". - +// // rawResponseHeaderKey is a header key attached to either a request or // response that signals a UnaryHandler to not encode an application error // inside a wirepb.Response object, instead marshalling the actual response. From ae2a78cff808f195135e90bc3c6fa57cf6c654cc Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 17:45:09 +0200 Subject: [PATCH 38/82] Fix lint --- transport/x/grpc/codec_test.go | 1 - transport/x/grpc/inbound.go | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/transport/x/grpc/codec_test.go b/transport/x/grpc/codec_test.go index baef9b4eb..610d30e91 100644 --- a/transport/x/grpc/codec_test.go +++ b/transport/x/grpc/codec_test.go @@ -60,7 +60,6 @@ func TestCustomCodecUnmarshalBytes(t *testing.T) { } func TestCustomCodecUnmarshalProtoMessage(t *testing.T) { - data := []byte("test") expectedValue := &yarpcproto.Oneway{true} data, err := proto.Marshal(expectedValue) assert.NoError(t, err) diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 43d43c144..1d63583e1 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -64,7 +64,7 @@ func (i *Inbound) Stop() error { // IsRunning implements transport.Lifecycle#IsRunning. func (i *Inbound) IsRunning() bool { - return i.IsRunning() + return i.once.IsRunning() } // SetRouter implements transport.Inbound#SetRouter. From 92c7315600848928bb5122e332bdfeba0ad1f34d Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 22:51:07 +0200 Subject: [PATCH 39/82] Basic grpc oneway implementation --- transport/x/grpc/codec.go | 12 ++++++++ transport/x/grpc/handlers.go | 15 ++++++++-- transport/x/grpc/outbound.go | 53 +++++++++++++++++++++++++----------- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/transport/x/grpc/codec.go b/transport/x/grpc/codec.go index c79e12468..f2b24d0cd 100644 --- a/transport/x/grpc/codec.go +++ b/transport/x/grpc/codec.go @@ -33,6 +33,12 @@ type customCodec struct{} // Marshal takes a proto.Message and marshals it, or // takes a []byte pointer and passes it through as a []byte. func (customCodec) Marshal(obj interface{}) ([]byte, error) { + // TODO: this is for oneway, are there side effects? + // should we just pass in a bytes array anyways? + // will this mess up with proto.Message? + if obj == nil { + return nil, nil + } switch value := obj.(type) { case proto.Message: return proto.Marshal(value) @@ -46,6 +52,12 @@ func (customCodec) Marshal(obj interface{}) ([]byte, error) { // Unmarshal takes a proto.Message and unmarshals it, or // takes a []byte pointer and writes it to v. func (customCodec) Unmarshal(data []byte, obj interface{}) error { + // TODO: this is for oneway, are there side effects? + // should we just pass in a bytes array anyways? + // will this mess up with proto.Message? + if obj == nil { + return nil + } switch value := obj.(type) { case proto.Message: return proto.Unmarshal(data, value) diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index abbf00886..95f436203 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -62,7 +62,6 @@ func (m *methodHandler) handle( if err != nil { return nil, err } - //log.Printf("%+v\n", transportRequest) if interceptor != nil { return interceptor( ctx, @@ -136,6 +135,8 @@ func (m *methodHandler) call(ctx context.Context, transportRequest *transport.Re switch handlerSpec.Type() { case transport.Unary: return m.callUnary(ctx, transportRequest, handlerSpec.Unary()) + case transport.Oneway: + return nil, m.callOneway(ctx, transportRequest, handlerSpec.Oneway()) default: return nil, errors.UnsupportedTypeError{"grpc", handlerSpec.Type().String()} } @@ -151,6 +152,16 @@ func (m *methodHandler) callUnary(ctx context.Context, transportRequest *transpo err := transport.DispatchUnaryHandler(ctx, unaryHandler, time.Now(), transportRequest, responseWriter) err = errors.CombineErrors(err, grpc.SendHeader(ctx, responseWriter.md)) data := responseWriter.Bytes() - //log.Printf("%s %v\n", string(data), err) return &data, err } + +func (m *methodHandler) callOneway(ctx context.Context, transportRequest *transport.Request, onewayHandler transport.OnewayHandler) error { + go func() { + // TODO: http propagates this on a span + // we need to handle this error + // TODO: spinning up a new goroutine for every request + // is potentially a memory leak + _ = transport.DispatchOnewayHandler(ctx, onewayHandler, transportRequest) + }() + return nil +} diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 7cd8f1641..abe3ec6a7 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -37,6 +37,7 @@ import ( ) var _ transport.UnaryOutbound = (*Outbound)(nil) +var _ transport.OnewayOutbound = (*Outbound)(nil) // Outbound is a transport.UnaryOutbound. type Outbound struct { @@ -73,39 +74,59 @@ func (o *Outbound) Transports() []transport.Transport { // Call implements transport.UnaryOutbound#Call. func (o *Outbound) Call(ctx context.Context, request *transport.Request) (*transport.Response, error) { + var responseBody []byte + responseMD := metadata.New(nil) + if err := o.invoke(ctx, request, &responseBody, &responseMD); err != nil { + return nil, err + } + responseHeaders, err := getApplicationHeaders(responseMD) + if err != nil { + return nil, err + } + return &transport.Response{ + Body: ioutil.NopCloser(bytes.NewBuffer(responseBody)), + Headers: responseHeaders, + }, nil +} + +// Call implements transport.OnewayOutbound#Call. +func (o *Outbound) CallOneway(ctx context.Context, request *transport.Request) (transport.Ack, error) { + if err := o.invoke(ctx, request, nil, nil); err != nil { + return nil, err + } + return time.Now(), nil +} + +func (o *Outbound) invoke( + ctx context.Context, + request *transport.Request, + responseBody *[]byte, + responseMD *metadata.MD, +) error { start := time.Now() md, err := requestToMetadata(request) if err != nil { - return nil, err + return err } requestBody, err := ioutil.ReadAll(request.Body) if err != nil { - return nil, err + return err } fullMethod, err := prodecureNameToFullMethod(request.Procedure) if err != nil { - return nil, err + return err } - var responseBody []byte - responseMD := metadata.New(nil) if err := grpc.Invoke( metadata.NewContext(ctx, md), fullMethod, &requestBody, - &responseBody, + responseBody, o.clientConn, - grpc.Header(&responseMD), + grpc.Header(responseMD), ); err != nil { - return nil, errorToGRPCError(ctx, request, start, err) + return errorToGRPCError(ctx, request, start, err) } - responseHeaders, err := getApplicationHeaders(responseMD) - if err != nil { - return nil, err - } - return &transport.Response{ - Body: ioutil.NopCloser(bytes.NewBuffer(responseBody)), - Headers: responseHeaders, - }, nil + return nil } func (o *Outbound) start() error { From 2a66063a3509f008e8b49c7d8db60f974e8f7b18 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 30 Mar 2017 23:04:51 +0200 Subject: [PATCH 40/82] Grpc oneway working --- internal/examples/protobuf/main.go | 23 ++++++++++++++++------- internal/testutils/testutils.go | 7 ++++--- transport/x/grpc/codec.go | 12 ------------ transport/x/grpc/outbound.go | 11 +++++++++-- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/internal/examples/protobuf/main.go b/internal/examples/protobuf/main.go index 5402bc0aa..1dcb045b9 100644 --- a/internal/examples/protobuf/main.go +++ b/internal/examples/protobuf/main.go @@ -93,9 +93,7 @@ func doClient( var err error if *flagGoogleGRPC { response, err = clients.KeyValueGRPCClient.GetValue(ctx, &examplepb.GetValueRequest{key}) - if err != nil { - err = errors.New(grpc.ErrorDesc(err)) - } + err = fromGRPCError(err) } else { response, err = clients.KeyValueYarpcClient.GetValue(ctx, &examplepb.GetValueRequest{key}) } @@ -120,9 +118,7 @@ func doClient( var err error if *flagGoogleGRPC { _, err = clients.KeyValueGRPCClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}) - if err != nil { - err = errors.New(grpc.ErrorDesc(err)) - } + err = fromGRPCError(err) } else { _, err = clients.KeyValueYarpcClient.SetValue(ctx, &examplepb.SetValueRequest{key, value}) } @@ -138,7 +134,13 @@ func doClient( value := args[0] ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() - if _, err := clients.SinkYarpcClient.Fire(ctx, &examplepb.FireRequest{value}); err != nil { + var err error + if *flagGoogleGRPC { + _, err = clients.SinkGRPCClient.Fire(ctx, &examplepb.FireRequest{value}) + } else { + _, err = clients.SinkYarpcClient.Fire(ctx, &examplepb.FireRequest{value}) + } + if err != nil { fmt.Printf("fire %s failed: %s\n", value, err.Error()) } if err := sinkYarpcServer.WaitFireDone(); err != nil { @@ -160,3 +162,10 @@ func doClient( } return scanner.Err() } + +func fromGRPCError(err error) error { + if err != nil { + return errors.New(grpc.ErrorDesc(err)) + } + return nil +} diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index 7a0534e1d..fc2afa496 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -135,7 +135,7 @@ func WithClientInfo(serviceName string, procedures []transport.Procedure, transp // NewClientDispatcher returns a new client Dispatcher. // -// HTTP always will be configured as an outbound for Oneway +// HTTP always will be configured as an outbound for Oneway unless using TransportTypeGRPC. func NewClientDispatcher(transportType TransportType, config *DispatcherConfig) (*yarpc.Dispatcher, error) { port, err := config.GetPort(transportType) if err != nil { @@ -160,8 +160,9 @@ func NewClientDispatcher(transportType TransportType, config *DispatcherConfig) onewayOutbound = httpOutbound unaryOutbound = httpOutbound case TransportTypeGRPC: - onewayOutbound = http.NewTransport().NewSingleOutbound(fmt.Sprintf("http://127.0.0.1:%d", httpPort)) - unaryOutbound = grpc.NewSingleOutbound(fmt.Sprintf("127.0.0.1:%d", port)) + grpcOutbound := grpc.NewSingleOutbound(fmt.Sprintf("127.0.0.1:%d", port)) + onewayOutbound = grpcOutbound + unaryOutbound = grpcOutbound default: return nil, fmt.Errorf("invalid TransportType: %v", transportType) } diff --git a/transport/x/grpc/codec.go b/transport/x/grpc/codec.go index f2b24d0cd..c79e12468 100644 --- a/transport/x/grpc/codec.go +++ b/transport/x/grpc/codec.go @@ -33,12 +33,6 @@ type customCodec struct{} // Marshal takes a proto.Message and marshals it, or // takes a []byte pointer and passes it through as a []byte. func (customCodec) Marshal(obj interface{}) ([]byte, error) { - // TODO: this is for oneway, are there side effects? - // should we just pass in a bytes array anyways? - // will this mess up with proto.Message? - if obj == nil { - return nil, nil - } switch value := obj.(type) { case proto.Message: return proto.Marshal(value) @@ -52,12 +46,6 @@ func (customCodec) Marshal(obj interface{}) ([]byte, error) { // Unmarshal takes a proto.Message and unmarshals it, or // takes a []byte pointer and writes it to v. func (customCodec) Unmarshal(data []byte, obj interface{}) error { - // TODO: this is for oneway, are there side effects? - // should we just pass in a bytes array anyways? - // will this mess up with proto.Message? - if obj == nil { - return nil - } switch value := obj.(type) { case proto.Message: return proto.Unmarshal(data, value) diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index abe3ec6a7..48fa781d4 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -91,7 +91,10 @@ func (o *Outbound) Call(ctx context.Context, request *transport.Request) (*trans // Call implements transport.OnewayOutbound#Call. func (o *Outbound) CallOneway(ctx context.Context, request *transport.Request) (transport.Ack, error) { - if err := o.invoke(ctx, request, nil, nil); err != nil { + // pass in dummy responseBody so code doesn't complain + // probably safer than doing nil check in codec + var responseBody []byte + if err := o.invoke(ctx, request, &responseBody, nil); err != nil { return nil, err } return time.Now(), nil @@ -116,13 +119,17 @@ func (o *Outbound) invoke( if err != nil { return err } + var callOptions []grpc.CallOption + if responseMD != nil { + callOptions = []grpc.CallOption{grpc.Header(responseMD)} + } if err := grpc.Invoke( metadata.NewContext(ctx, md), fullMethod, &requestBody, responseBody, o.clientConn, - grpc.Header(responseMD), + callOptions..., ); err != nil { return errorToGRPCError(ctx, request, start, err) } From a8de846d2e4ca3390468b384367c01db25982f5f Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Fri, 31 Mar 2017 01:10:34 +0200 Subject: [PATCH 41/82] Add grpc to example testing --- internal/examples/Makefile | 2 +- .../examples/json-keyvalue/client/main.go | 5 ++- .../examples/json-keyvalue/server/main.go | 2 ++ .../thrift-keyvalue/keyvalue/client/main.go | 9 ++++-- .../thrift-keyvalue/keyvalue/server/main.go | 6 ++-- transport/x/grpc/handlers.go | 3 +- transport/x/grpc/outbound.go | 2 +- transport/x/grpc/util.go | 19 +++++++++-- transport/x/grpc/util_test.go | 32 +++++++++++++++++++ 9 files changed, 68 insertions(+), 12 deletions(-) create mode 100644 transport/x/grpc/util_test.go diff --git a/internal/examples/Makefile b/internal/examples/Makefile index 20bfc1e26..1cab028fe 100644 --- a/internal/examples/Makefile +++ b/internal/examples/Makefile @@ -1,5 +1,5 @@ EXAMPLES = thrift-hello thrift-keyvalue json-keyvalue thrift-oneway -TRANSPORTS = http tchannel +TRANSPORTS = http tchannel grpc ENV = env CRAM = $(ENV)/bin/cram diff --git a/internal/examples/json-keyvalue/client/main.go b/internal/examples/json-keyvalue/client/main.go index b37a71fd2..7ff90ff51 100644 --- a/internal/examples/json-keyvalue/client/main.go +++ b/internal/examples/json-keyvalue/client/main.go @@ -35,6 +35,7 @@ import ( "go.uber.org/yarpc/encoding/json" "go.uber.org/yarpc/transport/http" "go.uber.org/yarpc/transport/tchannel" + "go.uber.org/yarpc/transport/x/grpc" ) type getRequest struct { @@ -72,7 +73,7 @@ func main() { outboundName := "" flag.StringVar( &outboundName, - "outbound", "", "name of the outbound to use (http/tchannel)", + "outbound", "", "name of the outbound to use (http/tchannel/grpc)", ) flag.Parse() @@ -89,6 +90,8 @@ func main() { outbound = httpTransport.NewSingleOutbound("http://127.0.0.1:24034") case "tchannel": outbound = tchannelTransport.NewSingleOutbound("localhost:28941") + case "grpc": + outbound = grpc.NewSingleOutbound("127.0.0.1:24038") default: log.Fatalf("invalid outbound: %q\n", outboundName) } diff --git a/internal/examples/json-keyvalue/server/main.go b/internal/examples/json-keyvalue/server/main.go index 7cfce8849..9e4618e8f 100644 --- a/internal/examples/json-keyvalue/server/main.go +++ b/internal/examples/json-keyvalue/server/main.go @@ -31,6 +31,7 @@ import ( "go.uber.org/yarpc/encoding/json" "go.uber.org/yarpc/transport/http" "go.uber.org/yarpc/transport/tchannel" + "go.uber.org/yarpc/transport/x/grpc" ) type getRequest struct { @@ -84,6 +85,7 @@ func main() { Inbounds: yarpc.Inbounds{ tchannelTransport.NewInbound(), httpTransport.NewInbound(":24034"), + grpc.NewInbound(":24038"), }, InboundMiddleware: yarpc.InboundMiddleware{ Unary: requestLogInboundMiddleware{}, diff --git a/internal/examples/thrift-keyvalue/keyvalue/client/main.go b/internal/examples/thrift-keyvalue/keyvalue/client/main.go index fc9326ac0..8702e20f1 100644 --- a/internal/examples/thrift-keyvalue/keyvalue/client/main.go +++ b/internal/examples/thrift-keyvalue/keyvalue/client/main.go @@ -36,13 +36,14 @@ import ( "go.uber.org/yarpc/api/transport" "go.uber.org/yarpc/transport/http" "go.uber.org/yarpc/transport/tchannel" + "go.uber.org/yarpc/transport/x/grpc" ) func main() { outboundName := "" flag.StringVar( &outboundName, - "outbound", "", "name of the outbound to use (http/tchannel)", + "outbound", "", "name of the outbound to use (http/tchannel/grpc)", ) flag.Parse() @@ -56,9 +57,11 @@ func main() { var outbound transport.UnaryOutbound switch strings.ToLower(outboundName) { case "http": - outbound = httpTransport.NewSingleOutbound("http://127.0.0.1:24034") + outbound = httpTransport.NewSingleOutbound("http://127.0.0.1:24042") case "tchannel": - outbound = tchannelTransport.NewSingleOutbound("localhost:28941") + outbound = tchannelTransport.NewSingleOutbound("localhost:28945") + case "grpc": + outbound = grpc.NewSingleOutbound("127.0.0.1:24046") default: log.Fatalf("invalid outbound: %q\n", outboundName) } diff --git a/internal/examples/thrift-keyvalue/keyvalue/server/main.go b/internal/examples/thrift-keyvalue/keyvalue/server/main.go index 0b52c3196..ffed10e83 100644 --- a/internal/examples/thrift-keyvalue/keyvalue/server/main.go +++ b/internal/examples/thrift-keyvalue/keyvalue/server/main.go @@ -34,6 +34,7 @@ import ( "go.uber.org/yarpc" "go.uber.org/yarpc/transport/http" "go.uber.org/yarpc/transport/tchannel" + "go.uber.org/yarpc/transport/x/grpc" ) type handler struct { @@ -68,7 +69,7 @@ func main() { }() tchannelTransport, err := tchannel.NewChannelTransport( tchannel.ServiceName("keyvalue"), - tchannel.ListenAddr(":28941"), + tchannel.ListenAddr(":28945"), ) if err != nil { log.Fatal(err) @@ -79,7 +80,8 @@ func main() { Name: "keyvalue", Inbounds: yarpc.Inbounds{ tchannelTransport.NewInbound(), - httpTransport.NewInbound(":24034"), + httpTransport.NewInbound(":24042"), + grpc.NewInbound(":24046"), }, }) diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index 95f436203..553bfef8b 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -28,7 +28,6 @@ import ( "go.uber.org/yarpc/api/transport" "go.uber.org/yarpc/encoding/x/protobuf" "go.uber.org/yarpc/internal/errors" - "go.uber.org/yarpc/internal/procedure" "go.uber.org/yarpc/internal/request" "golang.org/x/net/context" @@ -113,7 +112,7 @@ func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func Caller: caller, Encoding: encoding, Service: m.procedureServiceName, - Procedure: procedure.ToName(m.serviceName, m.methodName), + Procedure: procedureToName(m.serviceName, m.methodName), Headers: headers, Body: bytes.NewBuffer(data), } diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 48fa781d4..3c09a9a82 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -115,7 +115,7 @@ func (o *Outbound) invoke( if err != nil { return err } - fullMethod, err := prodecureNameToFullMethod(request.Procedure) + fullMethod, err := procedureNameToFullMethod(request.Procedure) if err != nil { return err } diff --git a/transport/x/grpc/util.go b/transport/x/grpc/util.go index e04c4007c..3aec2a3ad 100644 --- a/transport/x/grpc/util.go +++ b/transport/x/grpc/util.go @@ -27,17 +27,22 @@ import ( "go.uber.org/yarpc/internal/procedure" ) +const defaultMethodName = "__default__" + func procedureNameToServiceNameMethodName(procedureName string) (string, string, error) { serviceName, methodName := procedure.FromName(procedureName) - if serviceName == "" || methodName == "" { + if serviceName == "" { return "", "", fmt.Errorf("invalid procedure name: %s", procedureName) } + if methodName == "" { + methodName = defaultMethodName + } // TODO: do we really need to do url.QueryEscape? // Are there consequences if there is a diff from the string and the url.QueryEscape string? return url.QueryEscape(serviceName), url.QueryEscape(methodName), nil } -func prodecureNameToFullMethod(procedureName string) (string, error) { +func procedureNameToFullMethod(procedureName string) (string, error) { serviceName, methodName, err := procedureNameToServiceNameMethodName(procedureName) if err != nil { return "", err @@ -46,5 +51,15 @@ func prodecureNameToFullMethod(procedureName string) (string, error) { } func toFullMethod(serviceName string, methodName string) string { + if methodName == "" { + methodName = defaultMethodName + } return fmt.Sprintf("/%s/%s", serviceName, methodName) } + +func procedureToName(serviceName string, methodName string) string { + if methodName == defaultMethodName { + return serviceName + } + return procedure.ToName(serviceName, methodName) +} diff --git a/transport/x/grpc/util_test.go b/transport/x/grpc/util_test.go new file mode 100644 index 000000000..e980cda8c --- /dev/null +++ b/transport/x/grpc/util_test.go @@ -0,0 +1,32 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDefaultMethodNameIsSameQueryEscaped(t *testing.T) { + assert.Equal(t, defaultMethodName, url.QueryEscape(defaultMethodName)) +} From fae985f637536e5358b27f885e0043796dd438da Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Fri, 31 Mar 2017 01:54:03 +0200 Subject: [PATCH 42/82] More example tests --- Makefile | 2 +- internal/examples/Makefile | 30 ++++++++++++++++---- internal/examples/protobuf/service-test.yaml | 4 ++- internal/service-test/main.go | 13 +++++++++ 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index e138d7403..f4debc233 100644 --- a/Makefile +++ b/Makefile @@ -280,7 +280,7 @@ goveralls: $(BIN)/goveralls .PHONY: examples examples: - RUN=$(RUN) $(MAKE) -C internal/examples + RUN=$(RUN) V=$(V) $(MAKE) -C internal/examples .PHONY: crossdock crossdock: $(DOCKER_COMPOSE) diff --git a/internal/examples/Makefile b/internal/examples/Makefile index c4d0a5297..3cdf563e1 100644 --- a/internal/examples/Makefile +++ b/internal/examples/Makefile @@ -1,15 +1,21 @@ +SERVICE_TEST_FLAGS := ifdef RUN -SERVICE_TEST := service-test --no-verify-output -else -SERVICE_TEST := service-test +SERVICE_TEST_FLAGS += --no-verify-output endif +ifdef V +SERVICE_TEST_FLAGS += --verbose +endif +SERVICE_TEST := service-test $(SERVICE_TEST_FLAGS) .PHONY: all all: \ json-keyvalue-http \ json-keyvalue-tchannel \ json-keyvalue-grpc \ - protobuf \ + protobuf-http \ + protobuf-tchannel \ + protobuf-grpc \ + protobuf-google-grpc \ thrift-hello \ thrift-keyvalue-http \ thrift-keyvalue-tchannel \ @@ -33,8 +39,20 @@ json-keyvalue-grpc: install TRANSPORT=grpc $(SERVICE_TEST) --dir json-keyvalue .PHONY: protobuf -protobuf: install - $(SERVICE_TEST) --dir protobuf +protobuf-http: install + TRANSPORT=http $(SERVICE_TEST) --dir protobuf + +.PHONY: protobuf +protobuf-tchannel: install + TRANSPORT=tchannel $(SERVICE_TEST) --dir protobuf + +.PHONY: protobuf +protobuf-grpc: install + TRANSPORT=grpc $(SERVICE_TEST) --dir protobuf + +.PHONY: protobuf +protobuf-google-grpc: install + TRANSPORT=grpc GOOGLE_GRPC=--google-grpc $(SERVICE_TEST) --dir protobuf .PHONY: thrift-hello thrift-hello: install diff --git a/internal/examples/protobuf/service-test.yaml b/internal/examples/protobuf/service-test.yaml index 9729fb957..d0e925082 100644 --- a/internal/examples/protobuf/service-test.yaml +++ b/internal/examples/protobuf/service-test.yaml @@ -1,4 +1,6 @@ -client_command: go run main.go +required_env_vars: + - TRANSPORT +client_command: go run main.go -outbound $TRANSPORT $GOOGLE_GRPC input: | get foo get foo diff --git a/internal/service-test/main.go b/internal/service-test/main.go index 35387573a..7276a101f 100644 --- a/internal/service-test/main.go +++ b/internal/service-test/main.go @@ -45,6 +45,7 @@ var ( flagConfigFilePath = flag.String("file", "service-test.yaml", "The configuration file to use relative to the context directory") flagTimeout = flag.Duration("timeout", 5*time.Second, "The time to wait until timing out") flagNoVerifyOutput = flag.Bool("no-verify-output", false, "Do not verify output and just run the commands") + flagVerbose = flag.Bool("verbose", false, "Enable verbose logging") errConfigNil = errors.New("config nil") errClientCommandNotSet = errors.New("config client_command not set") @@ -117,6 +118,7 @@ func do(contextDir string, configFilePath string, timeout time.Duration, verifyO errC := make(chan error) go func() { if serverCmd != nil { + logCmd(serverCmd) if err := serverCmd.Start(); err != nil { errC <- fmt.Errorf("error starting server: %v", err) return @@ -127,6 +129,7 @@ func do(contextDir string, configFilePath string, timeout time.Duration, verifyO <-time.After(time.Duration(config.SleepBeforeClientMs) * time.Millisecond) } } + logCmd(clientCmd) if err := clientCmd.Start(); err != nil { errC <- fmt.Errorf("error starting client: %v", err) return @@ -190,6 +193,10 @@ func killCmd(cmd *exec.Cmd) { } } +func logCmd(cmd *exec.Cmd) { + verboseLogPrintf("%s %s", cmd.Path, strings.Join(cmd.Args, " ")) +} + func validateRequiredEnvVars(requiredEnvVars []string) error { for _, requiredEnvVar := range requiredEnvVars { if os.Getenv(requiredEnvVar) == "" { @@ -256,3 +263,9 @@ func (l *lockedBuffer) Bytes() []byte { defer l.lock.RUnlock() return l.buffer.Bytes() } + +func verboseLogPrintf(format string, args ...interface{}) { + if *flagVerbose { + log.Printf(format, args...) + } +} From 1e2ac2507d5d6b83ce8a6f994d3659c08fc2b7b8 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Fri, 31 Mar 2017 01:55:15 +0200 Subject: [PATCH 43/82] Fix lint --- transport/x/grpc/outbound.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 3c09a9a82..e1a63c217 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -89,7 +89,7 @@ func (o *Outbound) Call(ctx context.Context, request *transport.Request) (*trans }, nil } -// Call implements transport.OnewayOutbound#Call. +// CallOneway implements transport.OnewayOutbound#Call. func (o *Outbound) CallOneway(ctx context.Context, request *transport.Request) (transport.Ack, error) { // pass in dummy responseBody so code doesn't complain // probably safer than doing nil check in codec From 0808d2ceb41fe3a1f49eeb4f5cfc4590cadec39a Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Fri, 31 Mar 2017 02:16:06 +0200 Subject: [PATCH 44/82] Add grpc-opentracing and glide update --- glide.lock | 8 ++++++-- glide.yaml | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/glide.lock b/glide.lock index 956691f14..4ca608e5d 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 2a6589dcf1dcd9411fc326e4fd9873c3930b840f7fea22f8af5146cb614a335d -updated: 2017-03-31T01:37:05.042854609+02:00 +hash: 2a32ebf8bef0c4d7146fea9186605b7eca36f3922485269929f777e8c8a050fc +updated: 2017-03-31T02:15:40.450735176+02:00 imports: - name: github.com/apache/thrift version: b2a4d4ae21c789b689dd162deb819665567f481c @@ -44,6 +44,10 @@ imports: - proto - name: github.com/gorilla/websocket version: 3ab3a8b8831546bd18fd182c20687ca853b2bb13 +- name: github.com/grpc-ecosystem/grpc-opentracing + version: ef7d6c8df7564c20c0a19ed9737f6d67990c61fa + subpackages: + - go/otgrpc - name: github.com/mattn/go-shellwords version: 005a0944d84452842197c2108bd9168ced206f78 - name: github.com/matttproud/golang_protobuf_extensions diff --git a/glide.yaml b/glide.yaml index 7f2cc1cab..707054fd8 100644 --- a/glide.yaml +++ b/glide.yaml @@ -10,6 +10,10 @@ import: version: ~0.4 - package: github.com/golang/mock version: master +- package: github.com/grpc-ecosystem/grpc-opentracing + version: master + subpackages: + - go/otgrpc - package: github.com/mattn/go-shellwords version: ^1 - package: github.com/mitchellh/mapstructure From 8a59b68216db695a03b042661ff2a999d2abad9b Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Fri, 31 Mar 2017 02:22:27 +0200 Subject: [PATCH 45/82] Add opentracing to grpc --- transport/x/grpc/inbound.go | 12 ++++++++++-- transport/x/grpc/outbound.go | 8 +++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 1d63583e1..8e9ef02e3 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -25,6 +25,9 @@ import ( "net" "sync" + "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" + "github.com/opentracing/opentracing-go" + "google.golang.org/grpc" "go.uber.org/yarpc/api/transport" @@ -89,8 +92,13 @@ func (i *Inbound) start() error { if err != nil { return err } - // TODO: want to support default codec - server := grpc.NewServer(grpc.CustomCodec(customCodec{})) + server := grpc.NewServer( + grpc.CustomCodec(customCodec{}), + // TODO: does this actually work for yarpc + // this needs a lot of review + // TODO: always global tracer? + grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(opentracing.GlobalTracer())), + ) for _, serviceDesc := range serviceDescs { server.RegisterService(serviceDesc, noopGrpcStruct{}) } diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index e1a63c217..a079948d4 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -27,6 +27,9 @@ import ( "sync" "time" + "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" + opentracing "github.com/opentracing/opentracing-go" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -141,8 +144,11 @@ func (o *Outbound) start() error { clientConn, err := grpc.Dial( o.address, grpc.WithInsecure(), - // TODO: want to support default codec grpc.WithCodec(customCodec{}), + // TODO: does this actually work for yarpc + // this needs a lot of review + // TODO: always global tracer? + grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer())), ) if err != nil { return err From d12b7705c44c82cbbf8c3c170d76657da14256db Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Sun, 2 Apr 2017 22:41:17 +0200 Subject: [PATCH 46/82] Merge --- internal/examples/Makefile | 7 ---- internal/service-test/main.go | 60 ----------------------------------- 2 files changed, 67 deletions(-) diff --git a/internal/examples/Makefile b/internal/examples/Makefile index a1186603f..ad1c64ac1 100644 --- a/internal/examples/Makefile +++ b/internal/examples/Makefile @@ -1,15 +1,8 @@ ifdef RUN -<<<<<<< HEAD -SERVICE_TEST_FLAGS += --no-verify-output -endif -ifdef V -SERVICE_TEST_FLAGS += --verbose -======= SERVICE_TEST_FLAGS += --no-validate-output endif ifdef V SERVICE_TEST_FLAGS += --debug ->>>>>>> dev endif SERVICE_TEST := service-test $(SERVICE_TEST_FLAGS) diff --git a/internal/service-test/main.go b/internal/service-test/main.go index 4f00f4521..798b0a69a 100644 --- a/internal/service-test/main.go +++ b/internal/service-test/main.go @@ -35,13 +35,8 @@ var ( flagDir = flag.String("dir", "", "The relative directory to operate from, defaults to current directory") flagConfigFilePath = flag.String("file", "service-test.yaml", "The configuration file to use relative to the context directory") flagTimeout = flag.Duration("timeout", 5*time.Second, "The time to wait until timing out") -<<<<<<< HEAD - flagNoVerifyOutput = flag.Bool("no-verify-output", false, "Do not verify output and just run the commands") - flagVerbose = flag.Bool("verbose", false, "Enable verbose logging") -======= flagNoVerifyOutput = flag.Bool("no-validate-output", false, "Do not validate output and just run the commands") flagDebug = flag.Bool("debug", false, "Log debug information") ->>>>>>> dev errSignal = errors.New("signal") ) @@ -81,38 +76,7 @@ func do( }() errC := make(chan error) -<<<<<<< HEAD - go func() { - if serverCmd != nil { - logCmd(serverCmd) - if err := serverCmd.Start(); err != nil { - errC <- fmt.Errorf("error starting server: %v", err) - return - } - // kind of weird that we can timeout too - // maybe add this to the timeout - if config.SleepBeforeClientMs != 0 { - <-time.After(time.Duration(config.SleepBeforeClientMs) * time.Millisecond) - } - } - logCmd(clientCmd) - if err := clientCmd.Start(); err != nil { - errC <- fmt.Errorf("error starting client: %v", err) - return - } - if err := clientCmd.Wait(); err != nil { - errC <- fmt.Errorf("error on client wait: %v", err) - return - } - if serverCmd != nil { - killCmd(serverCmd) - _ = serverCmd.Wait() - } - errC <- nil - }() -======= go func() { errC <- runCmds(cmds) }() ->>>>>>> dev select { case err := <-errC: if err != nil { @@ -135,35 +99,11 @@ func newCmds(configFilePath string, dir string, debug bool) ([]*cmd, error) { return config.Cmds(dir, debug) } -<<<<<<< HEAD -func cleanupCmds(cmds ...*exec.Cmd) { - for _, cmd := range cmds { - killCmd(cmd) - } -} - -func killCmd(cmd *exec.Cmd) { - if cmd != nil && cmd.Process != nil { - // https://medium.com/@felixge/killing-a-child-process-and-all-of-its-children-in-go-54079af94773 - _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) - } -} - -func logCmd(cmd *exec.Cmd) { - verboseLogPrintf("%s %s", cmd.Path, strings.Join(cmd.Args, " ")) -} - -func validateRequiredEnvVars(requiredEnvVars []string) error { - for _, requiredEnvVar := range requiredEnvVars { - if os.Getenv(requiredEnvVar) == "" { - return fmt.Errorf("environment variable %s must be set", requiredEnvVar) -======= func runCmds(cmds []*cmd) error { for i := 0; i < len(cmds)-1; i++ { cmd := cmds[i] if err := cmd.Start(); err != nil { return err ->>>>>>> dev } defer func() { cmd.Kill() From b88fc2f553534537abcd7d971631e1f49e706494 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Sun, 2 Apr 2017 22:44:26 +0200 Subject: [PATCH 47/82] Merge --- internal/examples/json-keyvalue/server/main.go | 2 +- internal/testutils/testutils.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/examples/json-keyvalue/server/main.go b/internal/examples/json-keyvalue/server/main.go index ba6490050..e74157aa7 100644 --- a/internal/examples/json-keyvalue/server/main.go +++ b/internal/examples/json-keyvalue/server/main.go @@ -104,7 +104,7 @@ func do() error { } inbound = tchannelTransport.NewInbound() case "grpc": - inbound = grpc.NewInbound(":24038") + inbound = grpc.NewInbound("127.0.0.1:24038") default: return fmt.Errorf("invalid inbound: %q", *flagInbound) } diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index f7cf7960b..17f3a1c1d 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -206,7 +206,7 @@ func NewServerDispatcher(procedures []transport.Procedure, config *DispatcherCon Inbounds: yarpc.Inbounds{ tchannelTransport.NewInbound(), http.NewTransport().NewInbound(fmt.Sprintf("127.0.0.1:%d", httpPort)), - grpc.NewInbound(fmt.Sprintf(":%d", grpcPort)), + grpc.NewInbound(fmt.Sprintf("127.0.0.1:%d", grpcPort)), }, }, ) From 552ffdf078ac535d56c76f745c14203d2b9ac683 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Sun, 2 Apr 2017 22:47:21 +0200 Subject: [PATCH 48/82] Merge --- internal/service-test/main.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/internal/service-test/main.go b/internal/service-test/main.go index 798b0a69a..cc6b92f62 100644 --- a/internal/service-test/main.go +++ b/internal/service-test/main.go @@ -131,9 +131,3 @@ func cleanupCmds(cmds []*cmd, validateOutput bool, err error) { cmd.Clean(validateOutput && err == nil) } } - -func verboseLogPrintf(format string, args ...interface{}) { - if *flagVerbose { - log.Printf(format, args...) - } -} From b820f6d7f8fef43e2e2a822cb84103d822d0a230 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Sun, 2 Apr 2017 22:56:19 +0200 Subject: [PATCH 49/82] Merge --- build/local.mk | 3 ++- internal/examples/Makefile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/build/local.mk b/build/local.mk index c7783d07a..4f2c6afe3 100644 --- a/build/local.mk +++ b/build/local.mk @@ -1,6 +1,7 @@ # Paths besides auto-detected generated files that should be excluded from # lint results. -LINT_EXCLUDES_EXTRAS = +LINT_EXCLUDES_EXTRAS = \ + transport/x/grpc/handlers.go # Regex for 'go vet' rules to ignore GOVET_IGNORE_RULES = \ diff --git a/internal/examples/Makefile b/internal/examples/Makefile index ad1c64ac1..dde42a461 100644 --- a/internal/examples/Makefile +++ b/internal/examples/Makefile @@ -81,7 +81,7 @@ thrift-keyvalue-tchannel: install build-thrift-keyvalue TRANSPORT=tchannel $(SERVICE_TEST) --dir thrift-keyvalue .PHONY: thrift-keyvalue-grpc -thrift-keyvalue-grpc: install +thrift-keyvalue-grpc: install build-thrift-keyvalue TRANSPORT=grpc $(SERVICE_TEST) --dir thrift-keyvalue .PHONY: thrift-oneway From c82f69f5247afa43ffdf72ffe576008d5d377e22 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 4 Apr 2017 14:29:18 +0200 Subject: [PATCH 50/82] Address Wills comments --- internal/examples/protobuf/main.go | 4 ++-- transport/x/grpc/handlers.go | 3 ++- transport/x/grpc/headers.go | 1 + transport/x/grpc/inbound.go | 7 +++---- transport/x/grpc/outbound.go | 1 + 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/internal/examples/protobuf/main.go b/internal/examples/protobuf/main.go index e5a0a35f1..f2d5f5900 100644 --- a/internal/examples/protobuf/main.go +++ b/internal/examples/protobuf/main.go @@ -31,11 +31,11 @@ import ( "strings" "time" - "google.golang.org/grpc" - "go.uber.org/yarpc/internal/examples/protobuf/example" "go.uber.org/yarpc/internal/examples/protobuf/examplepb" "go.uber.org/yarpc/internal/testutils" + + "google.golang.org/grpc" ) var flagOutbound = flag.String("outbound", "tchannel", "The outbound to use for unary calls") diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index 553bfef8b..98c657b65 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -25,6 +25,7 @@ import ( "fmt" "time" + "go.uber.org/multierr" "go.uber.org/yarpc/api/transport" "go.uber.org/yarpc/encoding/x/protobuf" "go.uber.org/yarpc/internal/errors" @@ -149,7 +150,7 @@ func (m *methodHandler) callUnary(ctx context.Context, transportRequest *transpo responseWriter := newResponseWriter() // TODO: always return data? err := transport.DispatchUnaryHandler(ctx, unaryHandler, time.Now(), transportRequest, responseWriter) - err = errors.CombineErrors(err, grpc.SendHeader(ctx, responseWriter.md)) + err = multierr.Append(err, grpc.SendHeader(ctx, responseWriter.md)) data := responseWriter.Bytes() return &data, err } diff --git a/transport/x/grpc/headers.go b/transport/x/grpc/headers.go index c9f2a6468..79fa1e1d5 100644 --- a/transport/x/grpc/headers.go +++ b/transport/x/grpc/headers.go @@ -25,6 +25,7 @@ import ( "strings" "go.uber.org/yarpc/api/transport" + "google.golang.org/grpc/metadata" ) diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 8e9ef02e3..60da69340 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -25,13 +25,12 @@ import ( "net" "sync" + "go.uber.org/yarpc/api/transport" + internalsync "go.uber.org/yarpc/internal/sync" + "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" "github.com/opentracing/opentracing-go" - "google.golang.org/grpc" - - "go.uber.org/yarpc/api/transport" - internalsync "go.uber.org/yarpc/internal/sync" ) var ( diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index a079948d4..7d84a2df3 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -114,6 +114,7 @@ func (o *Outbound) invoke( if err != nil { return err } + // TODO: use pooled buffers requestBody, err := ioutil.ReadAll(request.Body) if err != nil { return err From 87cf60ed183c1533e2235c395641fe11a307f762 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 5 Apr 2017 16:13:36 +0200 Subject: [PATCH 51/82] Add tracer as an option for both grpc inbound and outbound --- transport/x/grpc/inbound.go | 19 +++++++------ transport/x/grpc/options.go | 52 ++++++++++++++++++++++++++++++++++++ transport/x/grpc/outbound.go | 17 ++++++------ 3 files changed, 69 insertions(+), 19 deletions(-) create mode 100644 transport/x/grpc/options.go diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 60da69340..3c78f0a20 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -29,7 +29,6 @@ import ( internalsync "go.uber.org/yarpc/internal/sync" "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" - "github.com/opentracing/opentracing-go" "google.golang.org/grpc" ) @@ -42,16 +41,17 @@ var ( // Inbound is a grpc transport.Inbound. type Inbound struct { - once internalsync.LifecycleOnce - lock sync.Mutex - address string - router transport.Router - server *grpc.Server + once internalsync.LifecycleOnce + lock sync.Mutex + address string + transportOptions *transportOptions + router transport.Router + server *grpc.Server } // NewInbound returns a new Inbound for the given address. -func NewInbound(address string) *Inbound { - return &Inbound{internalsync.Once(), sync.Mutex{}, address, nil, nil} +func NewInbound(address string, options ...TransportOption) *Inbound { + return &Inbound{internalsync.Once(), sync.Mutex{}, address, newTransportOptions(options), nil, nil} } // Start implements transport.Lifecycle#Start. @@ -95,8 +95,7 @@ func (i *Inbound) start() error { grpc.CustomCodec(customCodec{}), // TODO: does this actually work for yarpc // this needs a lot of review - // TODO: always global tracer? - grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(opentracing.GlobalTracer())), + grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(i.transportOptions.getTracer())), ) for _, serviceDesc := range serviceDescs { server.RegisterService(serviceDesc, noopGrpcStruct{}) diff --git a/transport/x/grpc/options.go b/transport/x/grpc/options.go new file mode 100644 index 000000000..46c592f63 --- /dev/null +++ b/transport/x/grpc/options.go @@ -0,0 +1,52 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import opentracing "github.com/opentracing/opentracing-go" + +// TransportOption is an option for a transport. +type TransportOption func(*transportOptions) + +// WithTracer specifies the tracer to use. +func WithTracer(tracer opentracing.Tracer) TransportOption { + return func(transportOptions *transportOptions) { + transportOptions.tracer = tracer + } +} + +type transportOptions struct { + tracer opentracing.Tracer +} + +func newTransportOptions(options []TransportOption) *transportOptions { + transportOptions := &transportOptions{} + for _, option := range options { + option(transportOptions) + } + return transportOptions +} + +func (t *transportOptions) getTracer() opentracing.Tracer { + if t.tracer == nil { + return opentracing.GlobalTracer() + } + return t.tracer +} diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 7d84a2df3..ac8411432 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -28,7 +28,6 @@ import ( "time" "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" - opentracing "github.com/opentracing/opentracing-go" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -44,15 +43,16 @@ var _ transport.OnewayOutbound = (*Outbound)(nil) // Outbound is a transport.UnaryOutbound. type Outbound struct { - once internalsync.LifecycleOnce - lock sync.Mutex - address string - clientConn *grpc.ClientConn + once internalsync.LifecycleOnce + lock sync.Mutex + address string + transportOptions *transportOptions + clientConn *grpc.ClientConn } // NewSingleOutbound returns a new Outbound for the given adrress. -func NewSingleOutbound(address string) *Outbound { - return &Outbound{internalsync.Once(), sync.Mutex{}, address, nil} +func NewSingleOutbound(address string, options ...TransportOption) *Outbound { + return &Outbound{internalsync.Once(), sync.Mutex{}, address, newTransportOptions(options), nil} } // Start implements transport.Lifecycle#Start. @@ -148,8 +148,7 @@ func (o *Outbound) start() error { grpc.WithCodec(customCodec{}), // TODO: does this actually work for yarpc // this needs a lot of review - // TODO: always global tracer? - grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(opentracing.GlobalTracer())), + grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(o.transportOptions.getTracer())), ) if err != nil { return err From 293b14ad351db13f86d189071fd0454cffcfe085 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 5 Apr 2017 16:20:58 +0200 Subject: [PATCH 52/82] Remove proto.Message from custom codec --- transport/x/grpc/codec.go | 26 +++++++------------------- transport/x/grpc/codec_test.go | 23 +---------------------- 2 files changed, 8 insertions(+), 41 deletions(-) diff --git a/transport/x/grpc/codec.go b/transport/x/grpc/codec.go index c79e12468..007a845c9 100644 --- a/transport/x/grpc/codec.go +++ b/transport/x/grpc/codec.go @@ -20,22 +20,14 @@ package grpc -import ( - "fmt" +import "fmt" - "github.com/gogo/protobuf/proto" -) - -// customCodec will either handle proto.Message objects, or -// pass bytes to/from the wire without modification. +// customCodec pass bytes to/from the wire without modification. type customCodec struct{} -// Marshal takes a proto.Message and marshals it, or -// takes a []byte pointer and passes it through as a []byte. +// Marshal takes a []byte pointer and passes it through as a []byte. func (customCodec) Marshal(obj interface{}) ([]byte, error) { switch value := obj.(type) { - case proto.Message: - return proto.Marshal(value) case *[]byte: return *value, nil default: @@ -43,12 +35,9 @@ func (customCodec) Marshal(obj interface{}) ([]byte, error) { } } -// Unmarshal takes a proto.Message and unmarshals it, or -// takes a []byte pointer and writes it to v. +// Unmarshal takes a []byte pointer and writes it to v. func (customCodec) Unmarshal(data []byte, obj interface{}) error { switch value := obj.(type) { - case proto.Message: - return proto.Unmarshal(data, value) case *[]byte: *value = data return nil @@ -58,11 +47,10 @@ func (customCodec) Unmarshal(data []byte, obj interface{}) error { } func (customCodec) String() string { - // TODO: we might have to fake this as proto - // this is hacky - return "custom" + // TODO: faking this as proto to be compatible with existing grpc clients + return "proto" } func newCustomCodecCastError(actualObject interface{}) error { - return fmt.Errorf("expected object of either type proto.Message or *[]byte but got %T", actualObject) + return fmt.Errorf("expected object to be of type *[]byte but got %T", actualObject) } diff --git a/transport/x/grpc/codec_test.go b/transport/x/grpc/codec_test.go index 610d30e91..afe1b2a18 100644 --- a/transport/x/grpc/codec_test.go +++ b/transport/x/grpc/codec_test.go @@ -23,9 +23,6 @@ package grpc import ( "testing" - "go.uber.org/yarpc/yarpcproto" - - "github.com/gogo/protobuf/proto" "github.com/stretchr/testify/assert" ) @@ -36,15 +33,6 @@ func TestCustomCodecMarshalBytes(t *testing.T) { assert.NoError(t, err) } -func TestCustomCodecMarshalProtoMessage(t *testing.T) { - value := &yarpcproto.Oneway{true} - expectedData, err := proto.Marshal(value) - assert.NoError(t, err) - data, err := customCodec{}.Marshal(value) - assert.Equal(t, expectedData, data) - assert.NoError(t, err) -} - func TestCustomCodecMarshalCastError(t *testing.T) { value := "test" data, err := customCodec{}.Marshal(&value) @@ -59,15 +47,6 @@ func TestCustomCodecUnmarshalBytes(t *testing.T) { assert.Equal(t, data, value) } -func TestCustomCodecUnmarshalProtoMessage(t *testing.T) { - expectedValue := &yarpcproto.Oneway{true} - data, err := proto.Marshal(expectedValue) - assert.NoError(t, err) - value := &yarpcproto.Oneway{} - assert.NoError(t, customCodec{}.Unmarshal(data, value)) - assert.Equal(t, expectedValue, value) -} - func TestCustomCodecUnmarshalCastError(t *testing.T) { var value string err := customCodec{}.Unmarshal([]byte("test"), &value) @@ -76,5 +55,5 @@ func TestCustomCodecUnmarshalCastError(t *testing.T) { } func TestCustomCodecString(t *testing.T) { - assert.Equal(t, "custom", customCodec{}.String()) + assert.Equal(t, "proto", customCodec{}.String()) } From 6ceae8a1ac789e77cd5f7dd5ac1058bc85267767 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 13:15:50 +0200 Subject: [PATCH 53/82] TestGetServiceDescs --- transport/x/grpc/inbound.go | 1 + transport/x/grpc/inbound_test.go | 202 +++++++++++++++++++++++++++++++ 2 files changed, 203 insertions(+) create mode 100644 transport/x/grpc/inbound_test.go diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 3c78f0a20..398340ef0 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -167,6 +167,7 @@ func getServiceNameAndMethodDesc(router transport.Router, procedure transport.Pr return serviceName, grpc.MethodDesc{ MethodName: methodName, // TODO: what if two procedures have the same serviceName and methodName, but a different service? + // TODO: should we handle procedure.Encoding somehow? Handler: newMethodHandler(procedure.Service, serviceName, methodName, router).handle, }, nil } diff --git a/transport/x/grpc/inbound_test.go b/transport/x/grpc/inbound_test.go new file mode 100644 index 000000000..8f28b559f --- /dev/null +++ b/transport/x/grpc/inbound_test.go @@ -0,0 +1,202 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "context" + "errors" + "fmt" + "testing" + + "go.uber.org/yarpc/api/transport" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestGetServiceDescs(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + Name string + Procedures []transport.Procedure + ServiceDescs []*grpc.ServiceDesc + }{ + { + Name: "Basic", + Procedures: []transport.Procedure{ + transport.Procedure{ + Name: "KeyValue::GetValue", + Service: "Example", + }, + }, + ServiceDescs: []*grpc.ServiceDesc{ + &grpc.ServiceDesc{ + ServiceName: "KeyValue", + Methods: []grpc.MethodDesc{ + grpc.MethodDesc{ + MethodName: "GetValue", + }, + }, + }, + }, + }, + { + Name: "Two Methods", + Procedures: []transport.Procedure{ + transport.Procedure{ + Name: "KeyValue::GetValue", + Service: "Example", + }, + transport.Procedure{ + Name: "KeyValue::SetValue", + Service: "Example", + }, + }, + ServiceDescs: []*grpc.ServiceDesc{ + &grpc.ServiceDesc{ + ServiceName: "KeyValue", + Methods: []grpc.MethodDesc{ + grpc.MethodDesc{ + MethodName: "GetValue", + }, + grpc.MethodDesc{ + MethodName: "SetValue", + }, + }, + }, + }, + }, + { + Name: "Two Services", + Procedures: []transport.Procedure{ + transport.Procedure{ + Name: "KeyValue::GetValue", + Service: "Example", + }, + transport.Procedure{ + Name: "Sink::GetValue", + Service: "Example", + }, + }, + ServiceDescs: []*grpc.ServiceDesc{ + &grpc.ServiceDesc{ + ServiceName: "KeyValue", + Methods: []grpc.MethodDesc{ + grpc.MethodDesc{ + MethodName: "GetValue", + }, + }, + }, + &grpc.ServiceDesc{ + ServiceName: "Sink", + Methods: []grpc.MethodDesc{ + grpc.MethodDesc{ + MethodName: "GetValue", + }, + }, + }, + }, + }, + { + Name: "Default Method Name", + Procedures: []transport.Procedure{ + transport.Procedure{ + Name: "Foo", + Service: "Example", + }, + }, + ServiceDescs: []*grpc.ServiceDesc{ + &grpc.ServiceDesc{ + ServiceName: "Foo", + Methods: []grpc.MethodDesc{ + grpc.MethodDesc{ + MethodName: defaultMethodName, + }, + }, + }, + }, + }, + } { + t.Run(tt.Name, func(t *testing.T) { + serviceDescs, err := getServiceDescs(newTestTransportRouter(tt.Procedures)) + require.NoError(t, err) + testServiceDescServiceNamesCovered(t, serviceDescs, tt.ServiceDescs) + for _, expectedServiceDesc := range tt.ServiceDescs { + serviceDesc := testFindServiceDesc(t, serviceDescs, expectedServiceDesc.ServiceName) + testServiceDescMethodNamesCovered(t, serviceDesc, expectedServiceDesc) + } + }) + } +} + +type testTransportRouter struct { + procedures []transport.Procedure +} + +func newTestTransportRouter(procedures []transport.Procedure) *testTransportRouter { + return &testTransportRouter{procedures} +} + +func (t *testTransportRouter) Procedures() []transport.Procedure { + return t.procedures +} + +func (t *testTransportRouter) Choose(_ context.Context, _ *transport.Request) (transport.HandlerSpec, error) { + return transport.HandlerSpec{}, errors.New("not implemented") +} + +func testFindServiceDesc(t *testing.T, serviceDescs []*grpc.ServiceDesc, serviceName string) *grpc.ServiceDesc { + for _, serviceDesc := range serviceDescs { + if serviceDesc.ServiceName == serviceName { + return serviceDesc + } + } + require.NoError(t, fmt.Errorf("could not find *grpc.ServiceDesc with ServiceName %s", serviceName)) + return nil +} + +func testServiceDescServiceNamesCovered(t *testing.T, serviceDescs []*grpc.ServiceDesc, expectedServiceDescs []*grpc.ServiceDesc) { + require.Equal(t, len(expectedServiceDescs), len(serviceDescs)) + m := make(map[string]bool) + n := make(map[string]bool) + for _, serviceDesc := range expectedServiceDescs { + m[serviceDesc.ServiceName] = true + } + for _, serviceDesc := range serviceDescs { + n[serviceDesc.ServiceName] = true + } + require.Equal(t, m, n) +} + +func testServiceDescMethodNamesCovered(t *testing.T, serviceDesc *grpc.ServiceDesc, expectedServiceDesc *grpc.ServiceDesc) { + methodDescs := serviceDesc.Methods + expectedMethodDescs := expectedServiceDesc.Methods + require.Equal(t, len(expectedMethodDescs), len(methodDescs)) + m := make(map[string]bool) + n := make(map[string]bool) + for _, methodDesc := range expectedMethodDescs { + m[methodDesc.MethodName] = true + } + for _, methodDesc := range methodDescs { + n[methodDesc.MethodName] = true + } + require.Equal(t, m, n) +} From daef54ef5642d898fccf7e1014a46d8a10b80fac Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 13:28:05 +0200 Subject: [PATCH 54/82] Blank import protobuf example in protobuf testing package --- encoding/x/protobuf/testing/testing.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/encoding/x/protobuf/testing/testing.go b/encoding/x/protobuf/testing/testing.go index 0a0003f90..2b35861b4 100644 --- a/encoding/x/protobuf/testing/testing.go +++ b/encoding/x/protobuf/testing/testing.go @@ -19,3 +19,6 @@ // THE SOFTWARE. package testing + +// blank import to make sure scripts.cover.sh picks this up with .Deps +import _ "go.uber.org/yarpc/internal/examples/protobuf/example" From 922da8795b29b6288d47c82cb3f8538bb19241b2 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 13:30:42 +0200 Subject: [PATCH 55/82] Gofmt --- transport/x/grpc/inbound_test.go | 34 ++++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/transport/x/grpc/inbound_test.go b/transport/x/grpc/inbound_test.go index 8f28b559f..b0b1a5b23 100644 --- a/transport/x/grpc/inbound_test.go +++ b/transport/x/grpc/inbound_test.go @@ -42,16 +42,16 @@ func TestGetServiceDescs(t *testing.T) { { Name: "Basic", Procedures: []transport.Procedure{ - transport.Procedure{ + { Name: "KeyValue::GetValue", Service: "Example", }, }, ServiceDescs: []*grpc.ServiceDesc{ - &grpc.ServiceDesc{ + { ServiceName: "KeyValue", Methods: []grpc.MethodDesc{ - grpc.MethodDesc{ + { MethodName: "GetValue", }, }, @@ -61,23 +61,23 @@ func TestGetServiceDescs(t *testing.T) { { Name: "Two Methods", Procedures: []transport.Procedure{ - transport.Procedure{ + { Name: "KeyValue::GetValue", Service: "Example", }, - transport.Procedure{ + { Name: "KeyValue::SetValue", Service: "Example", }, }, ServiceDescs: []*grpc.ServiceDesc{ - &grpc.ServiceDesc{ + { ServiceName: "KeyValue", Methods: []grpc.MethodDesc{ - grpc.MethodDesc{ + { MethodName: "GetValue", }, - grpc.MethodDesc{ + { MethodName: "SetValue", }, }, @@ -87,28 +87,28 @@ func TestGetServiceDescs(t *testing.T) { { Name: "Two Services", Procedures: []transport.Procedure{ - transport.Procedure{ + { Name: "KeyValue::GetValue", Service: "Example", }, - transport.Procedure{ + { Name: "Sink::GetValue", Service: "Example", }, }, ServiceDescs: []*grpc.ServiceDesc{ - &grpc.ServiceDesc{ + { ServiceName: "KeyValue", Methods: []grpc.MethodDesc{ - grpc.MethodDesc{ + { MethodName: "GetValue", }, }, }, - &grpc.ServiceDesc{ + { ServiceName: "Sink", Methods: []grpc.MethodDesc{ - grpc.MethodDesc{ + { MethodName: "GetValue", }, }, @@ -118,16 +118,16 @@ func TestGetServiceDescs(t *testing.T) { { Name: "Default Method Name", Procedures: []transport.Procedure{ - transport.Procedure{ + { Name: "Foo", Service: "Example", }, }, ServiceDescs: []*grpc.ServiceDesc{ - &grpc.ServiceDesc{ + { ServiceName: "Foo", Methods: []grpc.MethodDesc{ - grpc.MethodDesc{ + { MethodName: defaultMethodName, }, }, From 3ad0cefe3cc42f7cc9088d683764a337a86510f3 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 13:43:27 +0200 Subject: [PATCH 56/82] Procedure name tests --- transport/x/grpc/util_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/transport/x/grpc/util_test.go b/transport/x/grpc/util_test.go index e980cda8c..a77260a65 100644 --- a/transport/x/grpc/util_test.go +++ b/transport/x/grpc/util_test.go @@ -27,6 +27,38 @@ import ( "github.com/stretchr/testify/assert" ) +func TestProcedureNameFunctions(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + ProcedureName string + ServiceName string + MethodName string + FullMethod string + }{ + { + ProcedureName: "KeyValue::GetValue", + ServiceName: "KeyValue", + MethodName: "GetValue", + FullMethod: "/KeyValue/GetValue", + }, + { + ProcedureName: "foo", + ServiceName: "foo", + MethodName: "__default__", + FullMethod: "/foo/__default__", + }, + } { + t.Run(tt.ProcedureName, func(t *testing.T) { + serviceName, methodName, err := procedureNameToServiceNameMethodName(tt.ProcedureName) + assert.NoError(t, err) + assert.Equal(t, tt.ServiceName, serviceName) + assert.Equal(t, tt.MethodName, methodName) + assert.Equal(t, tt.ProcedureName, procedureToName(serviceName, methodName)) + assert.Equal(t, tt.FullMethod, toFullMethod(serviceName, methodName)) + }) + } +} + func TestDefaultMethodNameIsSameQueryEscaped(t *testing.T) { assert.Equal(t, defaultMethodName, url.QueryEscape(defaultMethodName)) } From bcec6cefe5924351a41adab43c31dbba571d9fe9 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 13:55:55 +0200 Subject: [PATCH 57/82] Add zap wrapper for grpclog --- transport/x/grpc/log.go | 55 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 transport/x/grpc/log.go diff --git a/transport/x/grpc/log.go b/transport/x/grpc/log.go new file mode 100644 index 000000000..816b443a6 --- /dev/null +++ b/transport/x/grpc/log.go @@ -0,0 +1,55 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "go.uber.org/zap" + "google.golang.org/grpc/grpclog" +) + +// SetLogger will set the given logger to be the logger for grpclog. +func SetLogger(logger *zap.Logger) { + grpclog.SetLogger(newLogger(logger)) +} + +type logger struct { + *zap.SugaredLogger +} + +func newLogger(l *zap.Logger) *logger { + return &logger{l.Sugar()} +} + +func (l *logger) Print(args ...interface{}) { + l.SugaredLogger.Info(args...) +} + +func (l *logger) Printf(format string, args ...interface{}) { + l.SugaredLogger.Infof(format, args...) +} + +func (l *logger) Println(args ...interface{}) { + l.SugaredLogger.Info(args...) +} + +func (l *logger) Fatalln(args ...interface{}) { + l.SugaredLogger.Fatal(args...) +} From 1e4d16930f2a60910db32ff25241859d09381984 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 14:23:20 +0200 Subject: [PATCH 58/82] Fix golint --- encoding/x/protobuf/testing/testing.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/encoding/x/protobuf/testing/testing.go b/encoding/x/protobuf/testing/testing.go index 2b35861b4..9d02f158c 100644 --- a/encoding/x/protobuf/testing/testing.go +++ b/encoding/x/protobuf/testing/testing.go @@ -20,5 +20,7 @@ package testing -// blank import to make sure scripts.cover.sh picks this up with .Deps -import _ "go.uber.org/yarpc/internal/examples/protobuf/example" +import ( + // this is to make sure scripts/cover.sh picks this up with .Deps + _ "go.uber.org/yarpc/internal/examples/protobuf/example" +) From f1c57aae78d1527db63663829544424a90c51227 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 15:09:22 +0200 Subject: [PATCH 59/82] Crossdock echo protobuf files --- internal/crossdock/crossdockpb/.nocover | 0 .../crossdock/crossdockpb/crossdock.pb.go | 674 ++++++++++++++++++ .../crossdockpb/crossdock.pb.yarpc.go | 110 +++ .../crossdock/crossdockpb/crossdock.proto | 17 + scripts/generate.sh | 2 + 5 files changed, 803 insertions(+) create mode 100644 internal/crossdock/crossdockpb/.nocover create mode 100644 internal/crossdock/crossdockpb/crossdock.pb.go create mode 100644 internal/crossdock/crossdockpb/crossdock.pb.yarpc.go create mode 100644 internal/crossdock/crossdockpb/crossdock.proto diff --git a/internal/crossdock/crossdockpb/.nocover b/internal/crossdock/crossdockpb/.nocover new file mode 100644 index 000000000..e69de29bb diff --git a/internal/crossdock/crossdockpb/crossdock.pb.go b/internal/crossdock/crossdockpb/crossdock.pb.go new file mode 100644 index 000000000..e7e448631 --- /dev/null +++ b/internal/crossdock/crossdockpb/crossdock.pb.go @@ -0,0 +1,674 @@ +// Code generated by protoc-gen-gogo. +// source: internal/crossdock/crossdockpb/crossdock.proto +// DO NOT EDIT! + +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +/* +Package crossdockpb is a generated protocol buffer package. + +It is generated from these files: + internal/crossdock/crossdockpb/crossdock.proto + +It has these top-level messages: + Ping + Pong +*/ +package crossdockpb + +import proto "github.com/gogo/protobuf/proto" +import fmt "fmt" +import math "math" + +import strings "strings" +import reflect "reflect" + +import ( + context "golang.org/x/net/context" + grpc "google.golang.org/grpc" +) + +import io "io" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package + +type Ping struct { + Beep string `protobuf:"bytes,1,opt,name=beep,proto3" json:"beep,omitempty"` +} + +func (m *Ping) Reset() { *m = Ping{} } +func (*Ping) ProtoMessage() {} +func (*Ping) Descriptor() ([]byte, []int) { return fileDescriptorCrossdock, []int{0} } + +func (m *Ping) GetBeep() string { + if m != nil { + return m.Beep + } + return "" +} + +type Pong struct { + Beep string `protobuf:"bytes,1,opt,name=beep,proto3" json:"beep,omitempty"` +} + +func (m *Pong) Reset() { *m = Pong{} } +func (*Pong) ProtoMessage() {} +func (*Pong) Descriptor() ([]byte, []int) { return fileDescriptorCrossdock, []int{1} } + +func (m *Pong) GetBeep() string { + if m != nil { + return m.Beep + } + return "" +} + +func init() { + proto.RegisterType((*Ping)(nil), "uber.yarpc.internal.crossdock.Ping") + proto.RegisterType((*Pong)(nil), "uber.yarpc.internal.crossdock.Pong") +} +func (this *Ping) Equal(that interface{}) bool { + if that == nil { + if this == nil { + return true + } + return false + } + + that1, ok := that.(*Ping) + if !ok { + that2, ok := that.(Ping) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + if this == nil { + return true + } + return false + } else if this == nil { + return false + } + if this.Beep != that1.Beep { + return false + } + return true +} +func (this *Pong) Equal(that interface{}) bool { + if that == nil { + if this == nil { + return true + } + return false + } + + that1, ok := that.(*Pong) + if !ok { + that2, ok := that.(Pong) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + if this == nil { + return true + } + return false + } else if this == nil { + return false + } + if this.Beep != that1.Beep { + return false + } + return true +} +func (this *Ping) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&crossdockpb.Ping{") + s = append(s, "Beep: "+fmt.Sprintf("%#v", this.Beep)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func (this *Pong) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 5) + s = append(s, "&crossdockpb.Pong{") + s = append(s, "Beep: "+fmt.Sprintf("%#v", this.Beep)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} +func valueToGoStringCrossdock(v interface{}, typ string) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// Client API for Echo service + +type EchoClient interface { + Echo(ctx context.Context, in *Ping, opts ...grpc.CallOption) (*Pong, error) +} + +type echoClient struct { + cc *grpc.ClientConn +} + +func NewEchoClient(cc *grpc.ClientConn) EchoClient { + return &echoClient{cc} +} + +func (c *echoClient) Echo(ctx context.Context, in *Ping, opts ...grpc.CallOption) (*Pong, error) { + out := new(Pong) + err := grpc.Invoke(ctx, "/uber.yarpc.internal.crossdock.Echo/Echo", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// Server API for Echo service + +type EchoServer interface { + Echo(context.Context, *Ping) (*Pong, error) +} + +func RegisterEchoServer(s *grpc.Server, srv EchoServer) { + s.RegisterService(&_Echo_serviceDesc, srv) +} + +func _Echo_Echo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Ping) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(EchoServer).Echo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/uber.yarpc.internal.crossdock.Echo/Echo", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(EchoServer).Echo(ctx, req.(*Ping)) + } + return interceptor(ctx, in, info, handler) +} + +var _Echo_serviceDesc = grpc.ServiceDesc{ + ServiceName: "uber.yarpc.internal.crossdock.Echo", + HandlerType: (*EchoServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Echo", + Handler: _Echo_Echo_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "internal/crossdock/crossdockpb/crossdock.proto", +} + +func (m *Ping) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Ping) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Beep) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintCrossdock(dAtA, i, uint64(len(m.Beep))) + i += copy(dAtA[i:], m.Beep) + } + return i, nil +} + +func (m *Pong) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Pong) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Beep) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintCrossdock(dAtA, i, uint64(len(m.Beep))) + i += copy(dAtA[i:], m.Beep) + } + return i, nil +} + +func encodeFixed64Crossdock(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Crossdock(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintCrossdock(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *Ping) Size() (n int) { + var l int + _ = l + l = len(m.Beep) + if l > 0 { + n += 1 + l + sovCrossdock(uint64(l)) + } + return n +} + +func (m *Pong) Size() (n int) { + var l int + _ = l + l = len(m.Beep) + if l > 0 { + n += 1 + l + sovCrossdock(uint64(l)) + } + return n +} + +func sovCrossdock(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozCrossdock(x uint64) (n int) { + return sovCrossdock(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (this *Ping) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&Ping{`, + `Beep:` + fmt.Sprintf("%v", this.Beep) + `,`, + `}`, + }, "") + return s +} +func (this *Pong) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&Pong{`, + `Beep:` + fmt.Sprintf("%v", this.Beep) + `,`, + `}`, + }, "") + return s +} +func valueToStringCrossdock(v interface{}) string { + rv := reflect.ValueOf(v) + if rv.IsNil() { + return "nil" + } + pv := reflect.Indirect(rv).Interface() + return fmt.Sprintf("*%v", pv) +} +func (m *Ping) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCrossdock + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Ping: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Ping: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Beep", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCrossdock + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCrossdock + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Beep = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCrossdock(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthCrossdock + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Pong) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCrossdock + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Pong: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Pong: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Beep", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowCrossdock + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthCrossdock + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Beep = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipCrossdock(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthCrossdock + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipCrossdock(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCrossdock + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCrossdock + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCrossdock + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthCrossdock + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowCrossdock + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipCrossdock(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthCrossdock = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowCrossdock = fmt.Errorf("proto: integer overflow") +) + +func init() { + proto.RegisterFile("internal/crossdock/crossdockpb/crossdock.proto", fileDescriptorCrossdock) +} + +var fileDescriptorCrossdock = []byte{ + // 186 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0xcb, 0xcc, 0x2b, 0x49, + 0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x4f, 0x2e, 0xca, 0x2f, 0x2e, 0x4e, 0xc9, 0x4f, 0xce, 0x46, 0xb0, + 0x0a, 0x92, 0x10, 0x6c, 0xbd, 0x82, 0xa2, 0xfc, 0x92, 0x7c, 0x21, 0xd9, 0xd2, 0xa4, 0xd4, 0x22, + 0xbd, 0xca, 0xc4, 0xa2, 0x82, 0x64, 0xb8, 0x56, 0x3d, 0xb8, 0x22, 0x25, 0x29, 0x2e, 0x96, 0x80, + 0xcc, 0xbc, 0x74, 0x21, 0x21, 0x2e, 0x96, 0xa4, 0xd4, 0xd4, 0x02, 0x09, 0x46, 0x05, 0x46, 0x0d, + 0xce, 0x20, 0x30, 0x1b, 0x2c, 0x97, 0x8f, 0x5d, 0xce, 0x28, 0x82, 0x8b, 0xc5, 0x35, 0x39, 0x23, + 0x5f, 0x28, 0x00, 0x4a, 0x2b, 0xeb, 0xe1, 0xb5, 0x47, 0x0f, 0x64, 0x89, 0x14, 0x41, 0x45, 0xf9, + 0x79, 0xe9, 0x4e, 0x96, 0x17, 0x1e, 0xca, 0x31, 0xdc, 0x78, 0x28, 0xc7, 0xf0, 0xe1, 0xa1, 0x1c, + 0x63, 0xc3, 0x23, 0x39, 0xc6, 0x15, 0x8f, 0xe4, 0x18, 0x4f, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, + 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, 0x17, 0x8f, 0xe4, 0x18, 0x3e, 0x3c, 0x92, 0x63, 0x9c, 0xf0, + 0x58, 0x8e, 0x21, 0x8a, 0x1b, 0xc9, 0xef, 0x49, 0x6c, 0x60, 0x2f, 0x1b, 0x03, 0x02, 0x00, 0x00, + 0xff, 0xff, 0xd8, 0xb2, 0x97, 0x4f, 0x24, 0x01, 0x00, 0x00, +} diff --git a/internal/crossdock/crossdockpb/crossdock.pb.yarpc.go b/internal/crossdock/crossdockpb/crossdock.pb.yarpc.go new file mode 100644 index 000000000..8a1b94d2e --- /dev/null +++ b/internal/crossdock/crossdockpb/crossdock.pb.yarpc.go @@ -0,0 +1,110 @@ +// Code generated by protoc-gen-yarpc-go +// source: internal/crossdock/crossdockpb/crossdock.proto +// DO NOT EDIT! + +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package crossdockpb + +import ( + "context" + + "github.com/gogo/protobuf/proto" + "go.uber.org/yarpc" + "go.uber.org/yarpc/api/transport" + "go.uber.org/yarpc/encoding/x/protobuf" +) + +// EchoYarpcClient is the yarpc client-side interface for the Echo service. +type EchoYarpcClient interface { + Echo(context.Context, *Ping, ...yarpc.CallOption) (*Pong, error) +} + +// NewEchoYarpcClient builds a new yarpc client for the Echo service. +func NewEchoYarpcClient(clientConfig transport.ClientConfig) EchoYarpcClient { + return &_EchoYarpcCaller{protobuf.NewClient("uber.yarpc.internal.crossdock.Echo", clientConfig)} +} + +// EchoYarpcServer is the yarpc server-side interface for the Echo service. +type EchoYarpcServer interface { + Echo(context.Context, *Ping) (*Pong, error) +} + +// BuildEchoYarpcProcedures prepares an implementation of the Echo service for yarpc registration. +func BuildEchoYarpcProcedures(server EchoYarpcServer) []transport.Procedure { + handler := &_EchoYarpcHandler{server} + return protobuf.BuildProcedures( + "uber.yarpc.internal.crossdock.Echo", + map[string]transport.UnaryHandler{ + "Echo": protobuf.NewUnaryHandler(handler.Echo, newEcho_EchoYarpcRequest), + }, + map[string]transport.OnewayHandler{}, + ) +} + +type _EchoYarpcCaller struct { + client protobuf.Client +} + +func (c *_EchoYarpcCaller) Echo(ctx context.Context, request *Ping, options ...yarpc.CallOption) (*Pong, error) { + responseMessage, err := c.client.Call(ctx, "Echo", request, newEcho_EchoYarpcResponse, options...) + if responseMessage == nil { + return nil, err + } + response, ok := responseMessage.(*Pong) + if !ok { + return nil, protobuf.CastError(emptyEcho_EchoYarpcResponse, responseMessage) + } + return response, err +} + +type _EchoYarpcHandler struct { + server EchoYarpcServer +} + +func (h *_EchoYarpcHandler) Echo(ctx context.Context, requestMessage proto.Message) (proto.Message, error) { + var request *Ping + var ok bool + if requestMessage != nil { + request, ok = requestMessage.(*Ping) + if !ok { + return nil, protobuf.CastError(emptyEcho_EchoYarpcRequest, requestMessage) + } + } + response, err := h.server.Echo(ctx, request) + if response == nil { + return nil, err + } + return response, err +} + +func newEcho_EchoYarpcRequest() proto.Message { + return &Ping{} +} + +func newEcho_EchoYarpcResponse() proto.Message { + return &Pong{} +} + +var ( + emptyEcho_EchoYarpcRequest = &Ping{} + emptyEcho_EchoYarpcResponse = &Pong{} +) diff --git a/internal/crossdock/crossdockpb/crossdock.proto b/internal/crossdock/crossdockpb/crossdock.proto new file mode 100644 index 000000000..1ba07fdb2 --- /dev/null +++ b/internal/crossdock/crossdockpb/crossdock.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package uber.yarpc.internal.crossdock; + +option go_package = "crossdockpb"; + +message Ping { + string beep = 1; +} + +message Pong { + string beep = 1; +} + +service Echo { + rpc Echo(Ping) returns (Pong); +} diff --git a/scripts/generate.sh b/scripts/generate.sh index 0e0fcb4a8..6179b1e62 100755 --- a/scripts/generate.sh +++ b/scripts/generate.sh @@ -87,6 +87,8 @@ protoc_go yarpcproto/yarpc.proto protoc_go encoding/x/protobuf/internal/wirepb/wire.proto protoc_go_grpc internal/examples/protobuf/examplepb/example.proto protoc_yarpc_go internal/examples/protobuf/examplepb/example.proto +protoc_go_grpc internal/crossdock/crossdockpb/crossdock.proto +protoc_yarpc_go internal/crossdock/crossdockpb/crossdock.proto touch encoding/x/protobuf/internal/wirepb/.nocover touch internal/crossdock/thrift/gen-go/echo/.nocover From 32ed9ce4cc30ef70bf5158c01fc1f6efef51df5b Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 15:22:10 +0200 Subject: [PATCH 60/82] Update crossdock proto files --- .../crossdock/crossdockpb/crossdock.pb.go | 41 ++++++++++--------- .../crossdock/crossdockpb/crossdock.proto | 2 +- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/internal/crossdock/crossdockpb/crossdock.pb.go b/internal/crossdock/crossdockpb/crossdock.pb.go index e7e448631..1174a32f6 100644 --- a/internal/crossdock/crossdockpb/crossdock.pb.go +++ b/internal/crossdock/crossdockpb/crossdock.pb.go @@ -75,16 +75,16 @@ func (m *Ping) GetBeep() string { } type Pong struct { - Beep string `protobuf:"bytes,1,opt,name=beep,proto3" json:"beep,omitempty"` + Boop string `protobuf:"bytes,1,opt,name=boop,proto3" json:"boop,omitempty"` } func (m *Pong) Reset() { *m = Pong{} } func (*Pong) ProtoMessage() {} func (*Pong) Descriptor() ([]byte, []int) { return fileDescriptorCrossdock, []int{1} } -func (m *Pong) GetBeep() string { +func (m *Pong) GetBoop() string { if m != nil { - return m.Beep + return m.Boop } return "" } @@ -148,7 +148,7 @@ func (this *Pong) Equal(that interface{}) bool { } else if this == nil { return false } - if this.Beep != that1.Beep { + if this.Boop != that1.Boop { return false } return true @@ -169,7 +169,7 @@ func (this *Pong) GoString() string { } s := make([]string, 0, 5) s = append(s, "&crossdockpb.Pong{") - s = append(s, "Beep: "+fmt.Sprintf("%#v", this.Beep)+",\n") + s = append(s, "Boop: "+fmt.Sprintf("%#v", this.Boop)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -293,11 +293,11 @@ func (m *Pong) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Beep) > 0 { + if len(m.Boop) > 0 { dAtA[i] = 0xa i++ - i = encodeVarintCrossdock(dAtA, i, uint64(len(m.Beep))) - i += copy(dAtA[i:], m.Beep) + i = encodeVarintCrossdock(dAtA, i, uint64(len(m.Boop))) + i += copy(dAtA[i:], m.Boop) } return i, nil } @@ -342,7 +342,7 @@ func (m *Ping) Size() (n int) { func (m *Pong) Size() (n int) { var l int _ = l - l = len(m.Beep) + l = len(m.Boop) if l > 0 { n += 1 + l + sovCrossdock(uint64(l)) } @@ -377,7 +377,7 @@ func (this *Pong) String() string { return "nil" } s := strings.Join([]string{`&Pong{`, - `Beep:` + fmt.Sprintf("%v", this.Beep) + `,`, + `Boop:` + fmt.Sprintf("%v", this.Boop) + `,`, `}`, }, "") return s @@ -500,7 +500,7 @@ func (m *Pong) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Beep", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Boop", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -525,7 +525,7 @@ func (m *Pong) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Beep = string(dAtA[iNdEx:postIndex]) + m.Boop = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex default: iNdEx = preIndex @@ -658,17 +658,18 @@ func init() { } var fileDescriptorCrossdock = []byte{ - // 186 bytes of a gzipped FileDescriptorProto + // 193 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0xcb, 0xcc, 0x2b, 0x49, 0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x4f, 0x2e, 0xca, 0x2f, 0x2e, 0x4e, 0xc9, 0x4f, 0xce, 0x46, 0xb0, 0x0a, 0x92, 0x10, 0x6c, 0xbd, 0x82, 0xa2, 0xfc, 0x92, 0x7c, 0x21, 0xd9, 0xd2, 0xa4, 0xd4, 0x22, 0xbd, 0xca, 0xc4, 0xa2, 0x82, 0x64, 0xb8, 0x56, 0x3d, 0xb8, 0x22, 0x25, 0x29, 0x2e, 0x96, 0x80, 0xcc, 0xbc, 0x74, 0x21, 0x21, 0x2e, 0x96, 0xa4, 0xd4, 0xd4, 0x02, 0x09, 0x46, 0x05, 0x46, 0x0d, - 0xce, 0x20, 0x30, 0x1b, 0x2c, 0x97, 0x8f, 0x5d, 0xce, 0x28, 0x82, 0x8b, 0xc5, 0x35, 0x39, 0x23, - 0x5f, 0x28, 0x00, 0x4a, 0x2b, 0xeb, 0xe1, 0xb5, 0x47, 0x0f, 0x64, 0x89, 0x14, 0x41, 0x45, 0xf9, - 0x79, 0xe9, 0x4e, 0x96, 0x17, 0x1e, 0xca, 0x31, 0xdc, 0x78, 0x28, 0xc7, 0xf0, 0xe1, 0xa1, 0x1c, - 0x63, 0xc3, 0x23, 0x39, 0xc6, 0x15, 0x8f, 0xe4, 0x18, 0x4f, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, - 0x8e, 0xf1, 0xc1, 0x23, 0x39, 0xc6, 0x17, 0x8f, 0xe4, 0x18, 0x3e, 0x3c, 0x92, 0x63, 0x9c, 0xf0, - 0x58, 0x8e, 0x21, 0x8a, 0x1b, 0xc9, 0xef, 0x49, 0x6c, 0x60, 0x2f, 0x1b, 0x03, 0x02, 0x00, 0x00, - 0xff, 0xff, 0xd8, 0xb2, 0x97, 0x4f, 0x24, 0x01, 0x00, 0x00, + 0xce, 0x20, 0x30, 0x1b, 0x2c, 0x97, 0x0f, 0x95, 0xcb, 0xcf, 0x47, 0xc8, 0xe5, 0xe7, 0x17, 0x18, + 0x45, 0x70, 0xb1, 0xb8, 0x26, 0x67, 0xe4, 0x0b, 0x05, 0x40, 0x69, 0x65, 0x3d, 0xbc, 0xf6, 0xe8, + 0x81, 0x2c, 0x91, 0x22, 0xa8, 0x28, 0x3f, 0x2f, 0xdd, 0xc9, 0xf2, 0xc2, 0x43, 0x39, 0x86, 0x1b, + 0x0f, 0xe5, 0x18, 0x3e, 0x3c, 0x94, 0x63, 0x6c, 0x78, 0x24, 0xc7, 0xb8, 0xe2, 0x91, 0x1c, 0xe3, + 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0xf8, 0xe2, 0x91, 0x1c, + 0xc3, 0x87, 0x47, 0x72, 0x8c, 0x13, 0x1e, 0xcb, 0x31, 0x44, 0x71, 0x23, 0xf9, 0x3d, 0x89, 0x0d, + 0xec, 0x65, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x56, 0xe7, 0x08, 0x7f, 0x24, 0x01, 0x00, + 0x00, } diff --git a/internal/crossdock/crossdockpb/crossdock.proto b/internal/crossdock/crossdockpb/crossdock.proto index 1ba07fdb2..1b6d490f1 100644 --- a/internal/crossdock/crossdockpb/crossdock.proto +++ b/internal/crossdock/crossdockpb/crossdock.proto @@ -9,7 +9,7 @@ message Ping { } message Pong { - string beep = 1; + string boop = 1; } service Echo { From 48a227dd669a004f5f7420bcc5ff86ea21351374 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 16:13:44 +0200 Subject: [PATCH 61/82] Basic protobuf crossdock tests --- docker-compose.yml | 3 ++ .../crossdock/client/dispatcher/dispatcher.go | 5 +- internal/crossdock/client/echo/behavior.go | 6 ++- internal/crossdock/client/echo/protobuf.go | 53 +++++++++++++++++++ internal/crossdock/client/params/constants.go | 9 ++-- internal/crossdock/client/start.go | 1 + internal/crossdock/server/yarpc/echo.go | 11 ++++ internal/crossdock/server/yarpc/server.go | 3 ++ 8 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 internal/crossdock/client/echo/protobuf.go diff --git a/docker-compose.yml b/docker-compose.yml index 289509481..d8a2f63ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,8 @@ services: - AXIS_CLIENT_ONEWAY=go - AXIS_SERVER_ONEWAY=go - AXIS_TRANSPORT_ONEWAY=http,redis,cherami + - AXIS_PROTOBUF_CLIENT=go + - AXIS_PROTOBUF_SERVER=go # Transports available to the ctxpropagation behavior for multihop # requests. @@ -43,6 +45,7 @@ services: - BEHAVIOR_RAW=client,server,transport - BEHAVIOR_JSON=client,server,transport - BEHAVIOR_THRIFT=client,server,transport + - BEHAVIOR_PROTOBUF=protobuf_client,protobuf_server,transport - BEHAVIOR_HEADERS=client,server,transport,encoding - BEHAVIOR_ERRORS_HTTPCLIENT=errors_httpclient,server # BEHAVIOR_ERRORSHTTPIN TODO diff --git a/internal/crossdock/client/dispatcher/dispatcher.go b/internal/crossdock/client/dispatcher/dispatcher.go index f9b61d1e2..68b7f9231 100644 --- a/internal/crossdock/client/dispatcher/dispatcher.go +++ b/internal/crossdock/client/dispatcher/dispatcher.go @@ -41,7 +41,10 @@ import ( func Create(t crossdock.T) *yarpc.Dispatcher { fatals := crossdock.Fatals(t) - server := t.Param(params.Server) + server := t.Param(params.ProtobufServer) + if server == "" { + server = t.Param(params.Server) + } fatals.NotEmpty(server, "server is required") var unaryOutbound transport.UnaryOutbound diff --git a/internal/crossdock/client/echo/behavior.go b/internal/crossdock/client/echo/behavior.go index 055fd3e4f..d21817d14 100644 --- a/internal/crossdock/client/echo/behavior.go +++ b/internal/crossdock/client/echo/behavior.go @@ -30,6 +30,10 @@ import ( func createEchoT(encoding string, t crossdock.T) crossdock.T { t.Tag("transport", t.Param(params.Transport)) t.Tag("encoding", encoding) - t.Tag("server", t.Param(params.Server)) + if t.Param(params.ProtobufServer) != "" { + t.Tag("server", t.Param(params.ProtobufServer)) + } else { + t.Tag("server", t.Param(params.Server)) + } return t } diff --git a/internal/crossdock/client/echo/protobuf.go b/internal/crossdock/client/echo/protobuf.go new file mode 100644 index 000000000..15df6dc17 --- /dev/null +++ b/internal/crossdock/client/echo/protobuf.go @@ -0,0 +1,53 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package echo + +import ( + "context" + "time" + + disp "go.uber.org/yarpc/internal/crossdock/client/dispatcher" + "go.uber.org/yarpc/internal/crossdock/client/random" + "go.uber.org/yarpc/internal/crossdock/crossdockpb" + + "github.com/crossdock/crossdock-go" +) + +// Protobuf implements the 'protobuf' behavior. +func Protobuf(t crossdock.T) { + t = createEchoT("protobuf", t) + fatals := crossdock.Fatals(t) + + dispatcher := disp.Create(t) + fatals.NoError(dispatcher.Start(), "could not start Dispatcher") + defer dispatcher.Stop() + + client := crossdockpb.NewEchoYarpcClient(dispatcher.ClientConfig("yarpc-test")) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + token := random.String(5) + + pong, err := client.Echo(ctx, &crossdockpb.Ping{Beep: token}) + + crossdock.Fatals(t).NoError(err, "call to Echo::echo failed: %v", err) + crossdock.Assert(t).Equal(token, pong.Boop, "server said: %v", pong.Boop) +} diff --git a/internal/crossdock/client/params/constants.go b/internal/crossdock/client/params/constants.go index 4e4878a2b..34542ef77 100644 --- a/internal/crossdock/client/params/constants.go +++ b/internal/crossdock/client/params/constants.go @@ -22,8 +22,9 @@ package params // Params used by behaviors const ( - Encoding = "encoding" - Server = "server" - HTTPServer = "httpserver" - Transport = "transport" + Encoding = "encoding" + Server = "server" + HTTPServer = "httpserver" + Transport = "transport" + ProtobufServer = "protobuf_server" ) diff --git a/internal/crossdock/client/start.go b/internal/crossdock/client/start.go index 5806be31b..f7c85564a 100644 --- a/internal/crossdock/client/start.go +++ b/internal/crossdock/client/start.go @@ -42,6 +42,7 @@ var behaviors = crossdock.Behaviors{ "raw": echo.Raw, "json": echo.JSON, "thrift": echo.Thrift, + "protobuf": echo.Protobuf, "headers": headers.Run, "errors_httpclient": errorshttpclient.Run, "errors_tchclient": errorstchclient.Run, diff --git a/internal/crossdock/server/yarpc/echo.go b/internal/crossdock/server/yarpc/echo.go index cffbc24a1..b15300f6d 100644 --- a/internal/crossdock/server/yarpc/echo.go +++ b/internal/crossdock/server/yarpc/echo.go @@ -24,6 +24,7 @@ import ( "context" "go.uber.org/yarpc" + "go.uber.org/yarpc/internal/crossdock/crossdockpb" "go.uber.org/yarpc/internal/crossdock/thrift/echo" ) @@ -62,3 +63,13 @@ func (EchoThrift) Echo(ctx context.Context, ping *echo.Ping) (*echo.Pong, error) } return &echo.Pong{Boop: ping.Beep}, nil } + +// EchoProtobuf implements the Protobuf Echo service. +type EchoProtobuf struct{} + +func (EchoProtobuf) Echo(_ context.Context, request *crossdockpb.Ping) (*crossdockpb.Pong, error) { + if request == nil { + return nil, nil + } + return &crossdockpb.Pong{Boop: request.Beep}, nil +} diff --git a/internal/crossdock/server/yarpc/server.go b/internal/crossdock/server/yarpc/server.go index d94f2ffc7..cac6c5d06 100644 --- a/internal/crossdock/server/yarpc/server.go +++ b/internal/crossdock/server/yarpc/server.go @@ -27,6 +27,7 @@ import ( "go.uber.org/yarpc" "go.uber.org/yarpc/encoding/json" "go.uber.org/yarpc/encoding/raw" + "go.uber.org/yarpc/internal/crossdock/crossdockpb" "go.uber.org/yarpc/internal/crossdock/thrift/echo/echoserver" "go.uber.org/yarpc/internal/crossdock/thrift/gauntlet/secondserviceserver" "go.uber.org/yarpc/internal/crossdock/thrift/gauntlet/thrifttestserver" @@ -87,4 +88,6 @@ func register(reg *yarpc.Dispatcher) { reg.Register(raw.Procedure("sleep/raw", SleepRaw)) reg.Register(raw.Procedure("waitfortimeout/raw", WaitForTimeoutRaw)) + + reg.Register(crossdockpb.BuildEchoYarpcProcedures(EchoProtobuf{})) } From c9d1e73b36c31fecdbf77c7f925304c7b0bd064a Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 18:23:31 +0200 Subject: [PATCH 62/82] Crossdock grpc testing --- Dockerfile.crossdock | 2 +- docker-compose.yml | 10 +++-- .../crossdock/client/dispatcher/dispatcher.go | 16 ++++++- internal/crossdock/client/echo/behavior.go | 11 +++-- internal/crossdock/client/echo/json.go | 9 +++- internal/crossdock/client/echo/protobuf.go | 9 +++- internal/crossdock/client/echo/raw.go | 9 +++- internal/crossdock/client/echo/thrift.go | 9 +++- internal/crossdock/client/grpc/grpc.go | 45 +++++++++++++++++++ internal/crossdock/client/params/constants.go | 11 ++--- internal/crossdock/client/start.go | 2 + internal/crossdock/server/yarpc/echo.go | 1 + internal/crossdock/server/yarpc/server.go | 2 + transport/x/grpc/util.go | 10 +++-- 14 files changed, 119 insertions(+), 27 deletions(-) create mode 100644 internal/crossdock/client/grpc/grpc.go diff --git a/Dockerfile.crossdock b/Dockerfile.crossdock index d094d09e0..0d6603edc 100644 --- a/Dockerfile.crossdock +++ b/Dockerfile.crossdock @@ -1,6 +1,6 @@ FROM golang:1.8.0 -EXPOSE 8080-8088 +EXPOSE 8080-8089 ENV SUPPRESS_DOCKER 1 WORKDIR /go/src/go.uber.org/yarpc ADD dockercrossdockdeps.mk /go/src/go.uber.org/yarpc/ diff --git a/docker-compose.yml b/docker-compose.yml index d8a2f63ad..5367fb9f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,8 +35,9 @@ services: - AXIS_CLIENT_ONEWAY=go - AXIS_SERVER_ONEWAY=go - AXIS_TRANSPORT_ONEWAY=http,redis,cherami - - AXIS_PROTOBUF_CLIENT=go - - AXIS_PROTOBUF_SERVER=go + - AXIS_GO_ENCODING=raw,json,thrift,protobuf + - AXIS_GO_CLIENT=go + - AXIS_GO_SERVER=go # Transports available to the ctxpropagation behavior for multihop # requests. @@ -45,7 +46,8 @@ services: - BEHAVIOR_RAW=client,server,transport - BEHAVIOR_JSON=client,server,transport - BEHAVIOR_THRIFT=client,server,transport - - BEHAVIOR_PROTOBUF=protobuf_client,protobuf_server,transport + - BEHAVIOR_PROTOBUF=go_client,go_server,transport + - BEHAVIOR_GRPC=go_client,go_server,go_encoding - BEHAVIOR_HEADERS=client,server,transport,encoding - BEHAVIOR_ERRORS_HTTPCLIENT=errors_httpclient,server # BEHAVIOR_ERRORSHTTPIN TODO @@ -74,7 +76,7 @@ services: context: . dockerfile: Dockerfile.crossdock ports: - - "8080-8088" + - "8080-8089" environment: - REDIS=enabled - CHERAMI=enabled diff --git a/internal/crossdock/client/dispatcher/dispatcher.go b/internal/crossdock/client/dispatcher/dispatcher.go index 68b7f9231..4d25eff80 100644 --- a/internal/crossdock/client/dispatcher/dispatcher.go +++ b/internal/crossdock/client/dispatcher/dispatcher.go @@ -31,6 +31,7 @@ import ( "go.uber.org/yarpc/transport/http" "go.uber.org/yarpc/transport/tchannel" "go.uber.org/yarpc/transport/x/cherami" + "go.uber.org/yarpc/transport/x/grpc" "go.uber.org/yarpc/transport/x/redis" "github.com/crossdock/crossdock-go" @@ -39,16 +40,25 @@ import ( // Create creates an RPC from the given parameters or fails the whole behavior. func Create(t crossdock.T) *yarpc.Dispatcher { + return CreateTransport(t, "") +} + +// CreateTransport creates an RPC from the given parameters or fails the whole behavior. +// +// If trans is non-empty, this will be used instead of the behavior transport. +func CreateTransport(t crossdock.T, trans string) *yarpc.Dispatcher { fatals := crossdock.Fatals(t) - server := t.Param(params.ProtobufServer) + server := t.Param(params.GoServer) if server == "" { server = t.Param(params.Server) } fatals.NotEmpty(server, "server is required") var unaryOutbound transport.UnaryOutbound - trans := t.Param(params.Transport) + if trans == "" { + trans = t.Param(params.Transport) + } switch trans { case "http": httpTransport := http.NewTransport() @@ -58,6 +68,8 @@ func Create(t crossdock.T) *yarpc.Dispatcher { fatals.NoError(err, "Failed to build ChannelTransport") unaryOutbound = tchannelTransport.NewSingleOutbound(server + ":8082") + case "grpc": + unaryOutbound = grpc.NewSingleOutbound(server + ":8089") default: fatals.Fail("", "unknown transport %q", trans) } diff --git a/internal/crossdock/client/echo/behavior.go b/internal/crossdock/client/echo/behavior.go index d21817d14..1d220730e 100644 --- a/internal/crossdock/client/echo/behavior.go +++ b/internal/crossdock/client/echo/behavior.go @@ -27,11 +27,14 @@ import ( ) // createEchoT tags the given T with the transport, encoding and server. -func createEchoT(encoding string, t crossdock.T) crossdock.T { - t.Tag("transport", t.Param(params.Transport)) +func createEchoT(encoding string, transport string, t crossdock.T) crossdock.T { + if transport == "" { + transport = t.Param(params.Transport) + } + t.Tag("transport", transport) t.Tag("encoding", encoding) - if t.Param(params.ProtobufServer) != "" { - t.Tag("server", t.Param(params.ProtobufServer)) + if t.Param(params.GoServer) != "" { + t.Tag("server", t.Param(params.GoServer)) } else { t.Tag("server", t.Param(params.Server)) } diff --git a/internal/crossdock/client/echo/json.go b/internal/crossdock/client/echo/json.go index e8072034d..4a5452faa 100644 --- a/internal/crossdock/client/echo/json.go +++ b/internal/crossdock/client/echo/json.go @@ -38,10 +38,15 @@ type jsonEcho struct { // JSON implements the 'json' behavior. func JSON(t crossdock.T) { - t = createEchoT("json", t) + JSONTransport(t, "") +} + +// JSONTransport implements the 'json' behavior for the given transport or behavior transport. +func JSONTransport(t crossdock.T, transport string) { + t = createEchoT("json", transport, t) fatals := crossdock.Fatals(t) - dispatcher := disp.Create(t) + dispatcher := disp.CreateTransport(t, transport) fatals.NoError(dispatcher.Start(), "could not start Dispatcher") defer dispatcher.Stop() diff --git a/internal/crossdock/client/echo/protobuf.go b/internal/crossdock/client/echo/protobuf.go index 15df6dc17..e8275f117 100644 --- a/internal/crossdock/client/echo/protobuf.go +++ b/internal/crossdock/client/echo/protobuf.go @@ -33,10 +33,15 @@ import ( // Protobuf implements the 'protobuf' behavior. func Protobuf(t crossdock.T) { - t = createEchoT("protobuf", t) + ProtobufTransport(t, "") +} + +// ProtobufTransport implements the 'protobuf' behavior for the given transport or behavior transport. +func ProtobufTransport(t crossdock.T, transport string) { + t = createEchoT("protobuf", transport, t) fatals := crossdock.Fatals(t) - dispatcher := disp.Create(t) + dispatcher := disp.CreateTransport(t, transport) fatals.NoError(dispatcher.Start(), "could not start Dispatcher") defer dispatcher.Stop() diff --git a/internal/crossdock/client/echo/raw.go b/internal/crossdock/client/echo/raw.go index 74ad13e9a..addcaaf43 100644 --- a/internal/crossdock/client/echo/raw.go +++ b/internal/crossdock/client/echo/raw.go @@ -34,10 +34,15 @@ import ( // Raw implements the 'raw' behavior. func Raw(t crossdock.T) { - t = createEchoT("raw", t) + RawTransport(t, "") +} + +// RawTransport implements the 'raw' behavior for the given transport or behavior transport. +func RawTransport(t crossdock.T, transport string) { + t = createEchoT("raw", transport, t) fatals := crossdock.Fatals(t) - dispatcher := disp.Create(t) + dispatcher := disp.CreateTransport(t, transport) fatals.NoError(dispatcher.Start(), "could not start Dispatcher") defer dispatcher.Stop() diff --git a/internal/crossdock/client/echo/thrift.go b/internal/crossdock/client/echo/thrift.go index de7fd5a93..2da00dc72 100644 --- a/internal/crossdock/client/echo/thrift.go +++ b/internal/crossdock/client/echo/thrift.go @@ -34,10 +34,15 @@ import ( // Thrift implements the 'thrift' behavior. func Thrift(t crossdock.T) { - t = createEchoT("thrift", t) + ThriftTransport(t, "") +} + +// ThriftTransport implements the 'thrift' behavior for the given transport or behavior transport. +func ThriftTransport(t crossdock.T, transport string) { + t = createEchoT("thrift", transport, t) fatals := crossdock.Fatals(t) - dispatcher := disp.Create(t) + dispatcher := disp.CreateTransport(t, transport) fatals.NoError(dispatcher.Start(), "could not start Dispatcher") defer dispatcher.Stop() diff --git a/internal/crossdock/client/grpc/grpc.go b/internal/crossdock/client/grpc/grpc.go new file mode 100644 index 000000000..eb322e116 --- /dev/null +++ b/internal/crossdock/client/grpc/grpc.go @@ -0,0 +1,45 @@ +// Copyright (c) 2017 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package grpc + +import ( + "go.uber.org/yarpc/internal/crossdock/client/echo" + "go.uber.org/yarpc/internal/crossdock/client/params" + + "github.com/crossdock/crossdock-go" +) + +var encodingToRunFunc = map[string]func(crossdock.T, string){ + "raw": echo.RawTransport, + "json": echo.JSONTransport, + "thrift": echo.ThriftTransport, + "protobuf": echo.ProtobufTransport, +} + +// Run starts the grpc behavior, testing grpc over encodings. +func Run(t crossdock.T) { + encoding := t.Param(params.GoEncoding) + f, ok := encodingToRunFunc[encoding] + if !ok { + crossdock.Fatals(t).Fail("unknown encoding", "%v", encoding) + } + f(t, "grpc") +} diff --git a/internal/crossdock/client/params/constants.go b/internal/crossdock/client/params/constants.go index 34542ef77..a01d3e291 100644 --- a/internal/crossdock/client/params/constants.go +++ b/internal/crossdock/client/params/constants.go @@ -22,9 +22,10 @@ package params // Params used by behaviors const ( - Encoding = "encoding" - Server = "server" - HTTPServer = "httpserver" - Transport = "transport" - ProtobufServer = "protobuf_server" + Encoding = "encoding" + Server = "server" + HTTPServer = "httpserver" + Transport = "transport" + GoEncoding = "go_encoding" + GoServer = "go_server" ) diff --git a/internal/crossdock/client/start.go b/internal/crossdock/client/start.go index f7c85564a..fd27eccdd 100644 --- a/internal/crossdock/client/start.go +++ b/internal/crossdock/client/start.go @@ -27,6 +27,7 @@ import ( "go.uber.org/yarpc/internal/crossdock/client/errorshttpclient" "go.uber.org/yarpc/internal/crossdock/client/errorstchclient" "go.uber.org/yarpc/internal/crossdock/client/gauntlet" + "go.uber.org/yarpc/internal/crossdock/client/grpc" "go.uber.org/yarpc/internal/crossdock/client/headers" "go.uber.org/yarpc/internal/crossdock/client/httpserver" "go.uber.org/yarpc/internal/crossdock/client/oneway" @@ -43,6 +44,7 @@ var behaviors = crossdock.Behaviors{ "json": echo.JSON, "thrift": echo.Thrift, "protobuf": echo.Protobuf, + "grpc": grpc.Run, "headers": headers.Run, "errors_httpclient": errorshttpclient.Run, "errors_tchclient": errorstchclient.Run, diff --git a/internal/crossdock/server/yarpc/echo.go b/internal/crossdock/server/yarpc/echo.go index b15300f6d..d59823db1 100644 --- a/internal/crossdock/server/yarpc/echo.go +++ b/internal/crossdock/server/yarpc/echo.go @@ -67,6 +67,7 @@ func (EchoThrift) Echo(ctx context.Context, ping *echo.Ping) (*echo.Pong, error) // EchoProtobuf implements the Protobuf Echo service. type EchoProtobuf struct{} +// Echo implements the Echo function for the Protobuf Echo service. func (EchoProtobuf) Echo(_ context.Context, request *crossdockpb.Ping) (*crossdockpb.Pong, error) { if request == nil { return nil, nil diff --git a/internal/crossdock/server/yarpc/server.go b/internal/crossdock/server/yarpc/server.go index cac6c5d06..515be05a9 100644 --- a/internal/crossdock/server/yarpc/server.go +++ b/internal/crossdock/server/yarpc/server.go @@ -33,6 +33,7 @@ import ( "go.uber.org/yarpc/internal/crossdock/thrift/gauntlet/thrifttestserver" "go.uber.org/yarpc/transport/http" "go.uber.org/yarpc/transport/tchannel" + "go.uber.org/yarpc/transport/x/grpc" ) var dispatcher *yarpc.Dispatcher @@ -53,6 +54,7 @@ func Start() { Inbounds: yarpc.Inbounds{ tchannelTransport.NewInbound(), httpTransport.NewInbound(":8081"), + grpc.NewInbound(":8089"), }, }) diff --git a/transport/x/grpc/util.go b/transport/x/grpc/util.go index 3aec2a3ad..23e445640 100644 --- a/transport/x/grpc/util.go +++ b/transport/x/grpc/util.go @@ -23,6 +23,7 @@ package grpc import ( "fmt" "net/url" + "strings" "go.uber.org/yarpc/internal/procedure" ) @@ -30,7 +31,8 @@ import ( const defaultMethodName = "__default__" func procedureNameToServiceNameMethodName(procedureName string) (string, string, error) { - serviceName, methodName := procedure.FromName(procedureName) + // TODO: this is hacky + serviceName, methodName := procedure.FromName(strings.Replace(procedureName, "/", "___", -1)) if serviceName == "" { return "", "", fmt.Errorf("invalid procedure name: %s", procedureName) } @@ -59,7 +61,9 @@ func toFullMethod(serviceName string, methodName string) string { func procedureToName(serviceName string, methodName string) string { if methodName == defaultMethodName { - return serviceName + // TODO: this is hacky + return strings.Replace(serviceName, "___", "/", -1) } - return procedure.ToName(serviceName, methodName) + // TODO: this is hacky + return strings.Replace(procedure.ToName(serviceName, methodName), "___", "/", -1) } From d6d663d49ef02f657fc20142924d6ccb942140a9 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 18:58:26 +0200 Subject: [PATCH 63/82] Start preparing crossdock for grpc oneway testing --- Dockerfile.crossdock | 2 +- docker-compose.yml | 3 ++- internal/crossdock/client/dispatcher/dispatcher.go | 2 ++ internal/crossdock/server/oneway/server.go | 11 +++++++++++ 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Dockerfile.crossdock b/Dockerfile.crossdock index 0d6603edc..00b2d2d63 100644 --- a/Dockerfile.crossdock +++ b/Dockerfile.crossdock @@ -1,6 +1,6 @@ FROM golang:1.8.0 -EXPOSE 8080-8089 +EXPOSE 8080-8090 ENV SUPPRESS_DOCKER 1 WORKDIR /go/src/go.uber.org/yarpc ADD dockercrossdockdeps.mk /go/src/go.uber.org/yarpc/ diff --git a/docker-compose.yml b/docker-compose.yml index 5367fb9f7..efb97f5f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -76,10 +76,11 @@ services: context: . dockerfile: Dockerfile.crossdock ports: - - "8080-8089" + - "8080-8090" environment: - REDIS=enabled - CHERAMI=enabled + - GRPC=enabled node: dns_search: . diff --git a/internal/crossdock/client/dispatcher/dispatcher.go b/internal/crossdock/client/dispatcher/dispatcher.go index 4d25eff80..5e4c057f7 100644 --- a/internal/crossdock/client/dispatcher/dispatcher.go +++ b/internal/crossdock/client/dispatcher/dispatcher.go @@ -117,6 +117,8 @@ func CreateOnewayDispatcher(t crossdock.T, handler raw.OnewayHandler) (*yarpc.Di outbound = transport.NewOutbound(cherami.OutboundConfig{ Destination: `/test/dest`}) + case "grpc": + outbound = grpc.NewSingleOutbound(server + ":8090") default: fatals.Fail("", "unknown transport %q", trans) } diff --git a/internal/crossdock/server/oneway/server.go b/internal/crossdock/server/oneway/server.go index a10707f13..3daf24aef 100644 --- a/internal/crossdock/server/oneway/server.go +++ b/internal/crossdock/server/oneway/server.go @@ -33,6 +33,7 @@ import ( "go.uber.org/yarpc/internal/crossdock/thrift/oneway/onewayserver" "go.uber.org/yarpc/transport/http" "go.uber.org/yarpc/transport/x/cherami" + "go.uber.org/yarpc/transport/x/grpc" "go.uber.org/yarpc/transport/x/redis" cherami_client "github.com/uber/cherami-client-go/client/cherami" @@ -64,6 +65,10 @@ func Start() { inbounds = append(inbounds, cheramiInboud) } + if useGrpc() { + inbounds = append(inbounds, grpc.NewInbound(":8090")) + } + dispatcher = yarpc.NewDispatcher(yarpc.Config{ Name: "oneway-server", Inbounds: inbounds, @@ -103,6 +108,12 @@ func useCherami() bool { return os.Getenv("CHERAMI") == "enabled" } +// useGrpc checks to see if a grpc server is expected to be +// available +func useGrpc() bool { + return os.Getenv("GRPC") == "enabled" +} + func initCheramiInbound() (*cherami.Inbound, error) { destination := `/test/dest` consumerGroup := `/test/dest_cg` From f908f8bb8b3a5108384996f87500200d50af3c85 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Thu, 6 Apr 2017 19:14:33 +0200 Subject: [PATCH 64/82] Add more headers for grpc and add test for procedure names --- transport/x/grpc/handlers.go | 9 ++++++++- transport/x/grpc/headers.go | 18 ++++++++++++++++++ transport/x/grpc/outbound.go | 6 ++++++ transport/x/grpc/util_test.go | 6 ++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index 98c657b65..ea244c412 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -101,6 +101,13 @@ func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func if encoding == "" { encoding = protobuf.Encoding } + service, err := getService(md) + if err != nil { + return nil, err + } + if service == "" { + service = m.procedureServiceName + } headers, err := getApplicationHeaders(md) if err != nil { return nil, err @@ -112,7 +119,7 @@ func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func transportRequest := &transport.Request{ Caller: caller, Encoding: encoding, - Service: m.procedureServiceName, + Service: service, Procedure: procedureToName(m.serviceName, m.methodName), Headers: headers, Body: bytes.NewBuffer(data), diff --git a/transport/x/grpc/headers.go b/transport/x/grpc/headers.go index 79fa1e1d5..2603900b2 100644 --- a/transport/x/grpc/headers.go +++ b/transport/x/grpc/headers.go @@ -35,6 +35,8 @@ const ( applicationHeaderPrefix = globalHeaderPrefix + "app-" callerHeader = reservedHeaderPrefix + "caller" encodingHeader = reservedHeaderPrefix + "encoding" + serviceHeader = reservedHeaderPrefix + "service" + procedureHeader = reservedHeaderPrefix + "procedure" ) func addCaller(md metadata.MD, caller string) error { @@ -45,6 +47,14 @@ func addEncoding(md metadata.MD, encoding transport.Encoding) error { return addToMetadata(md, encodingHeader, string(encoding)) } +func addService(md metadata.MD, service string) error { + return addToMetadata(md, serviceHeader, service) +} + +func addProcedure(md metadata.MD, procedure string) error { + return addToMetadata(md, procedureHeader, procedure) +} + // add headers into md as application headers // return error if md already has a key defined that is defined in headers func addApplicationHeaders(md metadata.MD, headers transport.Headers) error { @@ -68,6 +78,14 @@ func getEncoding(md metadata.MD) (transport.Encoding, error) { return transport.Encoding(encoding), nil } +func getService(md metadata.MD) (string, error) { + return getFromMetadata(md, serviceHeader) +} + +func getProcedure(md metadata.MD) (string, error) { + return getFromMetadata(md, procedureHeader) +} + // get application headers from md // return error if any application error has more than one value func getApplicationHeaders(md metadata.MD) (transport.Headers, error) { diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index ac8411432..d25af78fc 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -176,6 +176,12 @@ func requestToMetadata(request *transport.Request) (metadata.MD, error) { if err := addEncoding(md, request.Encoding); err != nil { return nil, err } + if err := addService(md, request.Service); err != nil { + return nil, err + } + if err := addProcedure(md, request.Procedure); err != nil { + return nil, err + } if err := addApplicationHeaders(md, request.Headers); err != nil { return nil, err } diff --git a/transport/x/grpc/util_test.go b/transport/x/grpc/util_test.go index a77260a65..19f18415d 100644 --- a/transport/x/grpc/util_test.go +++ b/transport/x/grpc/util_test.go @@ -47,6 +47,12 @@ func TestProcedureNameFunctions(t *testing.T) { MethodName: "__default__", FullMethod: "/foo/__default__", }, + { + ProcedureName: "foo/bar", + ServiceName: "foo___bar", + MethodName: "__default__", + FullMethod: "/foo___bar/__default__", + }, } { t.Run(tt.ProcedureName, func(t *testing.T) { serviceName, methodName, err := procedureNameToServiceNameMethodName(tt.ProcedureName) From 9f56edf1e3ebf37ba3d4aa96e33b3c2169853a22 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Fri, 7 Apr 2017 18:11:48 +0200 Subject: [PATCH 65/82] Options --- transport/x/grpc/handlers.go | 13 +++-- transport/x/grpc/inbound.go | 36 ++++++++------ transport/x/grpc/inbound_test.go | 4 +- transport/x/grpc/options.go | 85 ++++++++++++++++++++++++++------ transport/x/grpc/outbound.go | 16 +++--- 5 files changed, 112 insertions(+), 42 deletions(-) diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index ea244c412..1a446a177 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -41,6 +41,7 @@ type methodHandler struct { serviceName string methodName string router transport.Router + onewayErrorHandler func(error) } func newMethodHandler( @@ -48,8 +49,15 @@ func newMethodHandler( serviceName string, methodName string, router transport.Router, + onewayErrorHandler func(error), ) *methodHandler { - return &methodHandler{procedureServiceName, serviceName, methodName, router} + return &methodHandler{ + procedureServiceName, + serviceName, + methodName, + router, + onewayErrorHandler, + } } func (m *methodHandler) handle( @@ -165,10 +173,9 @@ func (m *methodHandler) callUnary(ctx context.Context, transportRequest *transpo func (m *methodHandler) callOneway(ctx context.Context, transportRequest *transport.Request, onewayHandler transport.OnewayHandler) error { go func() { // TODO: http propagates this on a span - // we need to handle this error // TODO: spinning up a new goroutine for every request // is potentially a memory leak - _ = transport.DispatchOnewayHandler(ctx, onewayHandler, transportRequest) + m.onewayErrorHandler(transport.DispatchOnewayHandler(ctx, onewayHandler, transportRequest)) }() return nil } diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 398340ef0..819497d92 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -41,17 +41,17 @@ var ( // Inbound is a grpc transport.Inbound. type Inbound struct { - once internalsync.LifecycleOnce - lock sync.Mutex - address string - transportOptions *transportOptions - router transport.Router - server *grpc.Server + once internalsync.LifecycleOnce + lock sync.Mutex + address string + inboundOptions *inboundOptions + router transport.Router + server *grpc.Server } // NewInbound returns a new Inbound for the given address. -func NewInbound(address string, options ...TransportOption) *Inbound { - return &Inbound{internalsync.Once(), sync.Mutex{}, address, newTransportOptions(options), nil, nil} +func NewInbound(address string, options ...InboundOption) *Inbound { + return &Inbound{internalsync.Once(), sync.Mutex{}, address, newInboundOptions(options), nil, nil} } // Start implements transport.Lifecycle#Start. @@ -87,7 +87,7 @@ func (i *Inbound) start() error { if i.router == nil { return errRouterNotSet } - serviceDescs, err := getServiceDescs(i.router) + serviceDescs, err := i.getServiceDescs() if err != nil { return err } @@ -95,7 +95,7 @@ func (i *Inbound) start() error { grpc.CustomCodec(customCodec{}), // TODO: does this actually work for yarpc // this needs a lot of review - grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(i.transportOptions.getTracer())), + grpc.UnaryInterceptor(otgrpc.OpenTracingServerInterceptor(i.inboundOptions.getTracer())), ) for _, serviceDesc := range serviceDescs { server.RegisterService(serviceDesc, noopGrpcStruct{}) @@ -130,15 +130,15 @@ func (i *Inbound) stop() error { return nil } -func getServiceDescs(router transport.Router) ([]*grpc.ServiceDesc, error) { +func (i *Inbound) getServiceDescs() ([]*grpc.ServiceDesc, error) { // TODO: router.Procedures() is not guaranteed to be immutable - procedures := router.Procedures() + procedures := i.router.Procedures() if len(procedures) == 0 { return nil, errRouterHasNoProcedures } serviceNameToServiceDesc := make(map[string]*grpc.ServiceDesc) for _, procedure := range procedures { - serviceName, methodDesc, err := getServiceNameAndMethodDesc(router, procedure) + serviceName, methodDesc, err := i.getServiceNameAndMethodDesc(procedure) if err != nil { return nil, err } @@ -159,7 +159,7 @@ func getServiceDescs(router transport.Router) ([]*grpc.ServiceDesc, error) { return serviceDescs, nil } -func getServiceNameAndMethodDesc(router transport.Router, procedure transport.Procedure) (string, grpc.MethodDesc, error) { +func (i *Inbound) getServiceNameAndMethodDesc(procedure transport.Procedure) (string, grpc.MethodDesc, error) { serviceName, methodName, err := procedureNameToServiceNameMethodName(procedure.Name) if err != nil { return "", grpc.MethodDesc{}, err @@ -168,7 +168,13 @@ func getServiceNameAndMethodDesc(router transport.Router, procedure transport.Pr MethodName: methodName, // TODO: what if two procedures have the same serviceName and methodName, but a different service? // TODO: should we handle procedure.Encoding somehow? - Handler: newMethodHandler(procedure.Service, serviceName, methodName, router).handle, + Handler: newMethodHandler( + procedure.Service, + serviceName, + methodName, + i.router, + i.inboundOptions.getOnewayErrorHandler(), + ).handle, }, nil } diff --git a/transport/x/grpc/inbound_test.go b/transport/x/grpc/inbound_test.go index b0b1a5b23..deeb99f53 100644 --- a/transport/x/grpc/inbound_test.go +++ b/transport/x/grpc/inbound_test.go @@ -136,7 +136,9 @@ func TestGetServiceDescs(t *testing.T) { }, } { t.Run(tt.Name, func(t *testing.T) { - serviceDescs, err := getServiceDescs(newTestTransportRouter(tt.Procedures)) + inbound := NewInbound("") + inbound.SetRouter(newTestTransportRouter(tt.Procedures)) + serviceDescs, err := inbound.getServiceDescs() require.NoError(t, err) testServiceDescServiceNamesCovered(t, serviceDescs, tt.ServiceDescs) for _, expectedServiceDesc := range tt.ServiceDescs { diff --git a/transport/x/grpc/options.go b/transport/x/grpc/options.go index 46c592f63..cc7e44005 100644 --- a/transport/x/grpc/options.go +++ b/transport/x/grpc/options.go @@ -20,33 +20,88 @@ package grpc -import opentracing "github.com/opentracing/opentracing-go" +import ( + "github.com/opentracing/opentracing-go" + "google.golang.org/grpc/grpclog" +) -// TransportOption is an option for a transport. -type TransportOption func(*transportOptions) +// InboundOption is an option for an inbound. +type InboundOption func(*inboundOptions) -// WithTracer specifies the tracer to use. -func WithTracer(tracer opentracing.Tracer) TransportOption { - return func(transportOptions *transportOptions) { - transportOptions.tracer = tracer +// OutboundOption is an option for an outbound. +type OutboundOption func(*outboundOptions) + +// WithInboundTracer specifies the tracer to use for an inbound. +func WithInboundTracer(tracer opentracing.Tracer) InboundOption { + return func(inboundOptions *inboundOptions) { + inboundOptions.tracer = tracer + } +} + +// WithInboundOnewayErrorHandler specifiec the error handler to use for an inbound. +// +// The default is to call grpclog.Print(err). +func WithInboundOnewayErrorHandler(onewayErrorHandler func(error)) InboundOption { + return func(inboundOptions *inboundOptions) { + inboundOptions.onewayErrorHandler = onewayErrorHandler + } +} + +// WithOutboundTracer specifies the tracer to use for an outbound. +func WithOutboundTracer(tracer opentracing.Tracer) OutboundOption { + return func(outboundOptions *outboundOptions) { + outboundOptions.tracer = tracer + } +} + +type inboundOptions struct { + tracer opentracing.Tracer + onewayErrorHandler func(error) +} + +func newInboundOptions(options []InboundOption) *inboundOptions { + inboundOptions := &inboundOptions{} + for _, option := range options { + option(inboundOptions) + } + return inboundOptions +} + +func (i *inboundOptions) getTracer() opentracing.Tracer { + if i.tracer == nil { + return opentracing.GlobalTracer() } + return i.tracer } -type transportOptions struct { +func (i *inboundOptions) getOnewayErrorHandler() func(error) { + if i.onewayErrorHandler == nil { + return defaultOnewayErrorHandler + } + return i.onewayErrorHandler +} + +type outboundOptions struct { tracer opentracing.Tracer } -func newTransportOptions(options []TransportOption) *transportOptions { - transportOptions := &transportOptions{} +func newOutboundOptions(options []OutboundOption) *outboundOptions { + outboundOptions := &outboundOptions{} for _, option := range options { - option(transportOptions) + option(outboundOptions) } - return transportOptions + return outboundOptions } -func (t *transportOptions) getTracer() opentracing.Tracer { - if t.tracer == nil { +func (o *outboundOptions) getTracer() opentracing.Tracer { + if o.tracer == nil { return opentracing.GlobalTracer() } - return t.tracer + return o.tracer +} + +func defaultOnewayErrorHandler(err error) { + if err != nil { + grpclog.Print(err) + } } diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index d25af78fc..036741bc4 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -43,16 +43,16 @@ var _ transport.OnewayOutbound = (*Outbound)(nil) // Outbound is a transport.UnaryOutbound. type Outbound struct { - once internalsync.LifecycleOnce - lock sync.Mutex - address string - transportOptions *transportOptions - clientConn *grpc.ClientConn + once internalsync.LifecycleOnce + lock sync.Mutex + address string + outboundOptions *outboundOptions + clientConn *grpc.ClientConn } // NewSingleOutbound returns a new Outbound for the given adrress. -func NewSingleOutbound(address string, options ...TransportOption) *Outbound { - return &Outbound{internalsync.Once(), sync.Mutex{}, address, newTransportOptions(options), nil} +func NewSingleOutbound(address string, options ...OutboundOption) *Outbound { + return &Outbound{internalsync.Once(), sync.Mutex{}, address, newOutboundOptions(options), nil} } // Start implements transport.Lifecycle#Start. @@ -148,7 +148,7 @@ func (o *Outbound) start() error { grpc.WithCodec(customCodec{}), // TODO: does this actually work for yarpc // this needs a lot of review - grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(o.transportOptions.getTracer())), + grpc.WithUnaryInterceptor(otgrpc.OpenTracingClientInterceptor(o.outboundOptions.getTracer())), ) if err != nil { return err From 7ec7ed1b508375c26e97746748e36e29230dba9b Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Fri, 7 Apr 2017 19:21:09 +0200 Subject: [PATCH 66/82] Crossdock oneway grpc but ctx propagation failing --- docker-compose.yml | 2 +- transport/x/grpc/handlers.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index efb97f5f5..b13ba9f76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,7 +34,7 @@ services: - AXIS_HTTPSERVER=go - AXIS_CLIENT_ONEWAY=go - AXIS_SERVER_ONEWAY=go - - AXIS_TRANSPORT_ONEWAY=http,redis,cherami + - AXIS_TRANSPORT_ONEWAY=grpc,http,redis,cherami - AXIS_GO_ENCODING=raw,json,thrift,protobuf - AXIS_GO_CLIENT=go - AXIS_GO_SERVER=go diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handlers.go index 1a446a177..bd6df0121 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handlers.go @@ -175,7 +175,11 @@ func (m *methodHandler) callOneway(ctx context.Context, transportRequest *transp // TODO: http propagates this on a span // TODO: spinning up a new goroutine for every request // is potentially a memory leak - m.onewayErrorHandler(transport.DispatchOnewayHandler(ctx, onewayHandler, transportRequest)) + // TODO: have to use context.Background() because context is cancelled in crossdock + // other transport implementation seem to create their own context for calls, need to understand better + // This will not propagate opentracing, for example + // Right now just letting context propagation test fail + m.onewayErrorHandler(transport.DispatchOnewayHandler(context.Background(), onewayHandler, transportRequest)) }() return nil } From 82422083ad00ba0fb056f8676618f9de2019a546 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Fri, 7 Apr 2017 20:17:43 +0200 Subject: [PATCH 67/82] Do WhenRunning check in grpc outbound --- transport/x/grpc/outbound.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 036741bc4..5b8eb73ff 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -77,6 +77,9 @@ func (o *Outbound) Transports() []transport.Transport { // Call implements transport.UnaryOutbound#Call. func (o *Outbound) Call(ctx context.Context, request *transport.Request) (*transport.Response, error) { + if err := o.once.WhenRunning(ctx); err != nil { + return nil, err + } var responseBody []byte responseMD := metadata.New(nil) if err := o.invoke(ctx, request, &responseBody, &responseMD); err != nil { @@ -94,6 +97,9 @@ func (o *Outbound) Call(ctx context.Context, request *transport.Request) (*trans // CallOneway implements transport.OnewayOutbound#Call. func (o *Outbound) CallOneway(ctx context.Context, request *transport.Request) (transport.Ack, error) { + if err := o.once.WhenRunning(ctx); err != nil { + return nil, err + } // pass in dummy responseBody so code doesn't complain // probably safer than doing nil check in codec var responseBody []byte From d608ee7320b14a7687581dc03c837a6631e41f8a Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Fri, 7 Apr 2017 20:28:05 +0200 Subject: [PATCH 68/82] Do not run oneway context propagation crossdock test on grpc --- docker-compose.yml | 3 ++- internal/crossdock/client/dispatcher/dispatcher.go | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index b13ba9f76..56061d457 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,7 @@ services: - AXIS_CLIENT_ONEWAY=go - AXIS_SERVER_ONEWAY=go - AXIS_TRANSPORT_ONEWAY=grpc,http,redis,cherami + - AXIS_TRANSPORT_ONEWAY_CTXPROPAGATION=http,redis,cherami - AXIS_GO_ENCODING=raw,json,thrift,protobuf - AXIS_GO_CLIENT=go - AXIS_GO_SERVER=go @@ -63,7 +64,7 @@ services: - BEHAVIOR_CTXPROPAGATION=ctxclient,ctxserver,transport,ctxavailabletransports - BEHAVIOR_APACHETHRIFT=apachethriftclient,apachethriftserver - BEHAVIOR_ONEWAY=client_oneway,server_oneway,transport_oneway,encoding - - BEHAVIOR_ONEWAY_CTXPROPAGATION=client_oneway,server_oneway,transport_oneway + - BEHAVIOR_ONEWAY_CTXPROPAGATION=client_oneway,server_oneway,transport_oneway_ctxpropagation - REPORT=compact diff --git a/internal/crossdock/client/dispatcher/dispatcher.go b/internal/crossdock/client/dispatcher/dispatcher.go index 5e4c057f7..baa34123f 100644 --- a/internal/crossdock/client/dispatcher/dispatcher.go +++ b/internal/crossdock/client/dispatcher/dispatcher.go @@ -96,6 +96,9 @@ func CreateOnewayDispatcher(t crossdock.T, handler raw.OnewayHandler) (*yarpc.Di var outbound transport.OnewayOutbound trans := t.Param("transport_oneway") + if trans == "" { + trans = t.Param("transport_oneway_ctxpropagation") + } switch trans { case "http": outbound = httpTransport.NewSingleOutbound(fmt.Sprintf("http://%s:8084", server)) From d61fd0f9fdea580cc413afeaaa3fa8156c3eb5fe Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Sat, 8 Apr 2017 15:59:05 +0200 Subject: [PATCH 69/82] Rename methodHandler to handler --- transport/x/grpc/{handlers.go => handler.go} | 44 ++++++++++---------- transport/x/grpc/inbound.go | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) rename transport/x/grpc/{handlers.go => handler.go} (77%) diff --git a/transport/x/grpc/handlers.go b/transport/x/grpc/handler.go similarity index 77% rename from transport/x/grpc/handlers.go rename to transport/x/grpc/handler.go index bd6df0121..58f8cc4fa 100644 --- a/transport/x/grpc/handlers.go +++ b/transport/x/grpc/handler.go @@ -36,7 +36,7 @@ import ( "google.golang.org/grpc/metadata" ) -type methodHandler struct { +type handler struct { procedureServiceName string serviceName string methodName string @@ -44,14 +44,14 @@ type methodHandler struct { onewayErrorHandler func(error) } -func newMethodHandler( +func newHandler( procedureServiceName string, serviceName string, methodName string, router transport.Router, onewayErrorHandler func(error), -) *methodHandler { - return &methodHandler{ +) *handler { + return &handler{ procedureServiceName, serviceName, methodName, @@ -60,13 +60,13 @@ func newMethodHandler( } } -func (m *methodHandler) handle( +func (h *handler) handle( server interface{}, ctx context.Context, decodeFunc func(interface{}) error, interceptor grpc.UnaryServerInterceptor, ) (interface{}, error) { - transportRequest, err := m.getTransportRequest(ctx, decodeFunc) + transportRequest, err := h.getTransportRequest(ctx, decodeFunc) if err != nil { return nil, err } @@ -76,21 +76,21 @@ func (m *methodHandler) handle( transportRequest, &grpc.UnaryServerInfo{ noopGrpcStruct{}, - m.getFullMethod(), + h.getFullMethod(), }, func(ctx context.Context, request interface{}) (interface{}, error) { transportRequest, ok := request.(*transport.Request) if !ok { return nil, fmt.Errorf("expected *transport.Request, got %T", request) } - return m.call(ctx, transportRequest) + return h.call(ctx, transportRequest) }, ) } - return m.call(ctx, transportRequest) + return h.call(ctx, transportRequest) } -func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func(interface{}) error) (*transport.Request, error) { +func (h *handler) getTransportRequest(ctx context.Context, decodeFunc func(interface{}) error) (*transport.Request, error) { md, ok := metadata.FromContext(ctx) if md == nil || !ok { return nil, fmt.Errorf("cannot get metadata from ctx: %v", ctx) @@ -100,7 +100,7 @@ func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func return nil, err } if caller == "" { - caller = m.serviceName + caller = h.serviceName } encoding, err := getEncoding(md) if err != nil { @@ -114,7 +114,7 @@ func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func return nil, err } if service == "" { - service = m.procedureServiceName + service = h.procedureServiceName } headers, err := getApplicationHeaders(md) if err != nil { @@ -128,7 +128,7 @@ func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func Caller: caller, Encoding: encoding, Service: service, - Procedure: procedureToName(m.serviceName, m.methodName), + Procedure: procedureToName(h.serviceName, h.methodName), Headers: headers, Body: bytes.NewBuffer(data), } @@ -138,26 +138,26 @@ func (m *methodHandler) getTransportRequest(ctx context.Context, decodeFunc func return transportRequest, nil } -func (m *methodHandler) getFullMethod() string { - return fmt.Sprintf("/%s/%s", m.serviceName, m.methodName) +func (h *handler) getFullMethod() string { + return fmt.Sprintf("/%s/%s", h.serviceName, h.methodName) } -func (m *methodHandler) call(ctx context.Context, transportRequest *transport.Request) (interface{}, error) { - handlerSpec, err := m.router.Choose(ctx, transportRequest) +func (h *handler) call(ctx context.Context, transportRequest *transport.Request) (interface{}, error) { + handlerSpec, err := h.router.Choose(ctx, transportRequest) if err != nil { return nil, err } switch handlerSpec.Type() { case transport.Unary: - return m.callUnary(ctx, transportRequest, handlerSpec.Unary()) + return h.callUnary(ctx, transportRequest, handlerSpec.Unary()) case transport.Oneway: - return nil, m.callOneway(ctx, transportRequest, handlerSpec.Oneway()) + return nil, h.callOneway(ctx, transportRequest, handlerSpec.Oneway()) default: return nil, errors.UnsupportedTypeError{"grpc", handlerSpec.Type().String()} } } -func (m *methodHandler) callUnary(ctx context.Context, transportRequest *transport.Request, unaryHandler transport.UnaryHandler) (interface{}, error) { +func (h *handler) callUnary(ctx context.Context, transportRequest *transport.Request, unaryHandler transport.UnaryHandler) (interface{}, error) { if err := request.ValidateUnaryContext(ctx); err != nil { return nil, err } @@ -170,7 +170,7 @@ func (m *methodHandler) callUnary(ctx context.Context, transportRequest *transpo return &data, err } -func (m *methodHandler) callOneway(ctx context.Context, transportRequest *transport.Request, onewayHandler transport.OnewayHandler) error { +func (h *handler) callOneway(ctx context.Context, transportRequest *transport.Request, onewayHandler transport.OnewayHandler) error { go func() { // TODO: http propagates this on a span // TODO: spinning up a new goroutine for every request @@ -179,7 +179,7 @@ func (m *methodHandler) callOneway(ctx context.Context, transportRequest *transp // other transport implementation seem to create their own context for calls, need to understand better // This will not propagate opentracing, for example // Right now just letting context propagation test fail - m.onewayErrorHandler(transport.DispatchOnewayHandler(context.Background(), onewayHandler, transportRequest)) + h.onewayErrorHandler(transport.DispatchOnewayHandler(context.Background(), onewayHandler, transportRequest)) }() return nil } diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 819497d92..e281309c9 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -168,7 +168,7 @@ func (i *Inbound) getServiceNameAndMethodDesc(procedure transport.Procedure) (st MethodName: methodName, // TODO: what if two procedures have the same serviceName and methodName, but a different service? // TODO: should we handle procedure.Encoding somehow? - Handler: newMethodHandler( + Handler: newHandler( procedure.Service, serviceName, methodName, From 70b259e3222473dae2f12051f6be354fe8691456 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Sat, 8 Apr 2017 16:16:12 +0200 Subject: [PATCH 70/82] Have grpc Inbound take a listener instead of an address --- internal/crossdock/server/oneway/server.go | 8 +++++++- internal/crossdock/server/yarpc/server.go | 7 ++++++- internal/examples/json-keyvalue/server/main.go | 7 ++++++- .../thrift-keyvalue/keyvalue/server/main.go | 7 ++++++- internal/testutils/testutils.go | 6 +++++- transport/x/grpc/inbound.go | 14 +++++--------- transport/x/grpc/inbound_test.go | 2 +- 7 files changed, 36 insertions(+), 15 deletions(-) diff --git a/internal/crossdock/server/oneway/server.go b/internal/crossdock/server/oneway/server.go index 3daf24aef..71291a14d 100644 --- a/internal/crossdock/server/oneway/server.go +++ b/internal/crossdock/server/oneway/server.go @@ -22,6 +22,7 @@ package oneway import ( "log" + "net" "os" "strings" "time" @@ -66,7 +67,12 @@ func Start() { } if useGrpc() { - inbounds = append(inbounds, grpc.NewInbound(":8090")) + listener, err := net.Listen("tcp", ":8090") + if err != nil { + log.Printf("err init grpc inbound %v\n", err) + } else { + inbounds = append(inbounds, grpc.NewInbound(listener)) + } } dispatcher = yarpc.NewDispatcher(yarpc.Config{ diff --git a/internal/crossdock/server/yarpc/server.go b/internal/crossdock/server/yarpc/server.go index 515be05a9..7c3a4e36c 100644 --- a/internal/crossdock/server/yarpc/server.go +++ b/internal/crossdock/server/yarpc/server.go @@ -23,6 +23,7 @@ package yarpc import ( "fmt" "log" + "net" "go.uber.org/yarpc" "go.uber.org/yarpc/encoding/json" @@ -47,6 +48,10 @@ func Start() { if err != nil { log.Panicf("failed to build ChannelTransport: %v", err) } + grpcListener, err := net.Listen("tcp", ":8089") + if err != nil { + log.Panic(err) + } httpTransport := http.NewTransport() dispatcher = yarpc.NewDispatcher(yarpc.Config{ @@ -54,7 +59,7 @@ func Start() { Inbounds: yarpc.Inbounds{ tchannelTransport.NewInbound(), httpTransport.NewInbound(":8081"), - grpc.NewInbound(":8089"), + grpc.NewInbound(grpcListener), }, }) diff --git a/internal/examples/json-keyvalue/server/main.go b/internal/examples/json-keyvalue/server/main.go index e74157aa7..a2098415c 100644 --- a/internal/examples/json-keyvalue/server/main.go +++ b/internal/examples/json-keyvalue/server/main.go @@ -25,6 +25,7 @@ import ( "flag" "fmt" "log" + "net" "strings" "sync" @@ -104,7 +105,11 @@ func do() error { } inbound = tchannelTransport.NewInbound() case "grpc": - inbound = grpc.NewInbound("127.0.0.1:24038") + listener, err := net.Listen("tcp", "127.0.0.1:24038") + if err != nil { + return err + } + inbound = grpc.NewInbound(listener) default: return fmt.Errorf("invalid inbound: %q", *flagInbound) } diff --git a/internal/examples/thrift-keyvalue/keyvalue/server/main.go b/internal/examples/thrift-keyvalue/keyvalue/server/main.go index 37fba7384..daa90704c 100644 --- a/internal/examples/thrift-keyvalue/keyvalue/server/main.go +++ b/internal/examples/thrift-keyvalue/keyvalue/server/main.go @@ -25,6 +25,7 @@ import ( "flag" "fmt" "log" + "net" gohttp "net/http" "strings" "sync" @@ -98,7 +99,11 @@ func do() error { } }() case "grpc": - inbound = grpc.NewInbound("127.0.0.1:24046") + listener, err := net.Listen("tcp", "127.0.0.1:24046") + if err != nil { + return err + } + inbound = grpc.NewInbound(listener) go func() { if err := gohttp.ListenAndServe(":3244", nil); err != nil { log.Fatal(err) diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index 8c770d147..c9cf99f3e 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -204,13 +204,17 @@ func NewServerDispatcher(procedures []transport.Procedure, config *DispatcherCon if err != nil { return nil, err } + grpcListener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", grpcPort)) + if err != nil { + return nil, err + } dispatcher := yarpc.NewDispatcher( yarpc.Config{ Name: config.GetServiceName(), Inbounds: yarpc.Inbounds{ tchannelTransport.NewInbound(), http.NewTransport().NewInbound(fmt.Sprintf("127.0.0.1:%d", httpPort)), - grpc.NewInbound(fmt.Sprintf("127.0.0.1:%d", grpcPort)), + grpc.NewInbound(grpcListener), }, }, ) diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index e281309c9..f2dcb070a 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -43,15 +43,15 @@ var ( type Inbound struct { once internalsync.LifecycleOnce lock sync.Mutex - address string + listener net.Listener inboundOptions *inboundOptions router transport.Router server *grpc.Server } -// NewInbound returns a new Inbound for the given address. -func NewInbound(address string, options ...InboundOption) *Inbound { - return &Inbound{internalsync.Once(), sync.Mutex{}, address, newInboundOptions(options), nil, nil} +// NewInbound returns a new Inbound for the given listener. +func NewInbound(listener net.Listener, options ...InboundOption) *Inbound { + return &Inbound{internalsync.Once(), sync.Mutex{}, listener, newInboundOptions(options), nil, nil} } // Start implements transport.Lifecycle#Start. @@ -100,10 +100,6 @@ func (i *Inbound) start() error { for _, serviceDesc := range serviceDescs { server.RegisterService(serviceDesc, noopGrpcStruct{}) } - listener, err := net.Listen("tcp", i.address) - if err != nil { - return err - } go func() { // TODO there should be some mechanism to block here // there is a race because the listener gets set in the grpc @@ -115,7 +111,7 @@ func (i *Inbound) start() error { // // TODO Server always returns a non-nil error but should // we do something with some or all errors? - _ = server.Serve(listener) + _ = server.Serve(i.listener) }() i.server = server return nil diff --git a/transport/x/grpc/inbound_test.go b/transport/x/grpc/inbound_test.go index deeb99f53..f13458c6f 100644 --- a/transport/x/grpc/inbound_test.go +++ b/transport/x/grpc/inbound_test.go @@ -136,7 +136,7 @@ func TestGetServiceDescs(t *testing.T) { }, } { t.Run(tt.Name, func(t *testing.T) { - inbound := NewInbound("") + inbound := NewInbound(nil) inbound.SetRouter(newTestTransportRouter(tt.Procedures)) serviceDescs, err := inbound.getServiceDescs() require.NoError(t, err) From b159f50e3e60b73664467136203ff9972bd77d90 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Mon, 10 Apr 2017 18:47:42 +0200 Subject: [PATCH 71/82] Fix lint --- build/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/local.mk b/build/local.mk index 4f2c6afe3..44c5a3346 100644 --- a/build/local.mk +++ b/build/local.mk @@ -1,7 +1,7 @@ # Paths besides auto-detected generated files that should be excluded from # lint results. LINT_EXCLUDES_EXTRAS = \ - transport/x/grpc/handlers.go + transport/x/grpc/handler.go # Regex for 'go vet' rules to ignore GOVET_IGNORE_RULES = \ From ac5cd336979ae559409bce54030e61555ffbaab3 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 11 Apr 2017 13:40:04 +0200 Subject: [PATCH 72/82] Add comment about response headers --- encoding/x/protobuf/inbound.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/encoding/x/protobuf/inbound.go b/encoding/x/protobuf/inbound.go index 5f4ae1f4c..259e81fdd 100644 --- a/encoding/x/protobuf/inbound.go +++ b/encoding/x/protobuf/inbound.go @@ -81,6 +81,13 @@ func (u *unaryHandler) Handle(ctx context.Context, transportRequest *transport.R } responseData = protoBuffer.Bytes() } + // We have to detect if our transport requires a raw response + // It is not possible to propagate this information on ctx with the current API + // we we attach this in the relevant transport (currently only gRPC) on the headers + // If we are sending a raw response back to a YARPC client, it needs to understand + // this is happening, so we attach the headers on the response as well + // Other clients (namely the existing gRPC clients outside of YARPC) understand + // that the response is the raw response. if isRawResponse(transportRequest.Headers) { responseWriter.AddHeaders(getRawResponseHeaders()) _, err := responseWriter.Write(responseData) From 9e8d197090dcab79344478c894d381d4975f9006 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 11 Apr 2017 13:43:50 +0200 Subject: [PATCH 73/82] Rename CreateTransport to CreateDispatcherForTransport --- internal/crossdock/client/dispatcher/dispatcher.go | 6 +++--- internal/crossdock/client/echo/json.go | 2 +- internal/crossdock/client/echo/protobuf.go | 2 +- internal/crossdock/client/echo/raw.go | 2 +- internal/crossdock/client/echo/thrift.go | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/crossdock/client/dispatcher/dispatcher.go b/internal/crossdock/client/dispatcher/dispatcher.go index baa34123f..785865dec 100644 --- a/internal/crossdock/client/dispatcher/dispatcher.go +++ b/internal/crossdock/client/dispatcher/dispatcher.go @@ -40,13 +40,13 @@ import ( // Create creates an RPC from the given parameters or fails the whole behavior. func Create(t crossdock.T) *yarpc.Dispatcher { - return CreateTransport(t, "") + return CreateDispatcherForTransport(t, "") } -// CreateTransport creates an RPC from the given parameters or fails the whole behavior. +// CreateDispatcherForTransport creates an RPC from the given parameters or fails the whole behavior. // // If trans is non-empty, this will be used instead of the behavior transport. -func CreateTransport(t crossdock.T, trans string) *yarpc.Dispatcher { +func CreateDispatcherForTransport(t crossdock.T, trans string) *yarpc.Dispatcher { fatals := crossdock.Fatals(t) server := t.Param(params.GoServer) diff --git a/internal/crossdock/client/echo/json.go b/internal/crossdock/client/echo/json.go index 4a5452faa..fa1aa0bad 100644 --- a/internal/crossdock/client/echo/json.go +++ b/internal/crossdock/client/echo/json.go @@ -46,7 +46,7 @@ func JSONTransport(t crossdock.T, transport string) { t = createEchoT("json", transport, t) fatals := crossdock.Fatals(t) - dispatcher := disp.CreateTransport(t, transport) + dispatcher := disp.CreateDispatcherForTransport(t, transport) fatals.NoError(dispatcher.Start(), "could not start Dispatcher") defer dispatcher.Stop() diff --git a/internal/crossdock/client/echo/protobuf.go b/internal/crossdock/client/echo/protobuf.go index e8275f117..57268fb11 100644 --- a/internal/crossdock/client/echo/protobuf.go +++ b/internal/crossdock/client/echo/protobuf.go @@ -41,7 +41,7 @@ func ProtobufTransport(t crossdock.T, transport string) { t = createEchoT("protobuf", transport, t) fatals := crossdock.Fatals(t) - dispatcher := disp.CreateTransport(t, transport) + dispatcher := disp.CreateDispatcherForTransport(t, transport) fatals.NoError(dispatcher.Start(), "could not start Dispatcher") defer dispatcher.Stop() diff --git a/internal/crossdock/client/echo/raw.go b/internal/crossdock/client/echo/raw.go index addcaaf43..a45fad125 100644 --- a/internal/crossdock/client/echo/raw.go +++ b/internal/crossdock/client/echo/raw.go @@ -42,7 +42,7 @@ func RawTransport(t crossdock.T, transport string) { t = createEchoT("raw", transport, t) fatals := crossdock.Fatals(t) - dispatcher := disp.CreateTransport(t, transport) + dispatcher := disp.CreateDispatcherForTransport(t, transport) fatals.NoError(dispatcher.Start(), "could not start Dispatcher") defer dispatcher.Stop() diff --git a/internal/crossdock/client/echo/thrift.go b/internal/crossdock/client/echo/thrift.go index 2da00dc72..caa58b183 100644 --- a/internal/crossdock/client/echo/thrift.go +++ b/internal/crossdock/client/echo/thrift.go @@ -42,7 +42,7 @@ func ThriftTransport(t crossdock.T, transport string) { t = createEchoT("thrift", transport, t) fatals := crossdock.Fatals(t) - dispatcher := disp.CreateTransport(t, transport) + dispatcher := disp.CreateDispatcherForTransport(t, transport) fatals.NoError(dispatcher.Start(), "could not start Dispatcher") defer dispatcher.Stop() From 210af5097411ad1aa080eb1adc2e240f1fd96c84 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 11 Apr 2017 13:55:08 +0200 Subject: [PATCH 74/82] Comment about fully-qualified protobuf names --- internal/protoplugin/protoplugin.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/protoplugin/protoplugin.go b/internal/protoplugin/protoplugin.go index fb2c5f9bf..ebeaaf84a 100644 --- a/internal/protoplugin/protoplugin.go +++ b/internal/protoplugin/protoplugin.go @@ -28,6 +28,9 @@ This was HEAVILY adapted from github.com/grpc-ecosystem/grpc-gateway/protoc-gen- Eventually, a rewrite of this to be simplier for what we need would be nice, but this was available to get us here, especially with handling go imports. + +Note that "FQMN", "FQSN", etc stand for "Fully Qualified Message Name", +"Fully Qualified Service Name", etc, which denotes the package and object name together. */ package protoplugin From 3d9f35164a0bf46f6d3fb746c4856db3cc2608da Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 11 Apr 2017 14:01:12 +0200 Subject: [PATCH 75/82] Link to issue about grpc codec --- transport/x/grpc/codec.go | 1 + 1 file changed, 1 insertion(+) diff --git a/transport/x/grpc/codec.go b/transport/x/grpc/codec.go index 007a845c9..c711ac986 100644 --- a/transport/x/grpc/codec.go +++ b/transport/x/grpc/codec.go @@ -48,6 +48,7 @@ func (customCodec) Unmarshal(data []byte, obj interface{}) error { func (customCodec) String() string { // TODO: faking this as proto to be compatible with existing grpc clients + // https://github.com/yarpc/yarpc-go/issues/911 return "proto" } From fb9f0e6ed2c702465559d467b1cd3477650d812a Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 11 Apr 2017 14:30:00 +0200 Subject: [PATCH 76/82] Add comment on grpc package --- transport/x/grpc/doc.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transport/x/grpc/doc.go b/transport/x/grpc/doc.go index d460585d7..6168170db 100644 --- a/transport/x/grpc/doc.go +++ b/transport/x/grpc/doc.go @@ -19,4 +19,6 @@ // THE SOFTWARE. // Package grpc implements the grpc transport. +// +// This package is experimental and should not be used in production. package grpc From 5529d7a6a4404a924701e1b9d50d596237899ee2 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 11 Apr 2017 15:32:14 +0200 Subject: [PATCH 77/82] More fixes --- encoding/x/protobuf/types.go | 6 +++++ transport/x/grpc/handler.go | 45 +++++++++++++++++++++--------------- transport/x/grpc/inbound.go | 1 + 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/encoding/x/protobuf/types.go b/encoding/x/protobuf/types.go index 5985362df..5e3e4040f 100644 --- a/encoding/x/protobuf/types.go +++ b/encoding/x/protobuf/types.go @@ -43,6 +43,12 @@ const ( // rawResponseHeaderKey is a header key attached to either a request or // response that signals a UnaryHandler to not encode an application error // inside a wirepb.Response object, instead marshalling the actual response. +// +// Note per the documentation on transport.Headers#With, the returned Header +// may not be the same as the input header, so the caller should always +// update the header with: +// +// header = protobuf.SetRawResponse(header) func SetRawResponse(headers transport.Headers) transport.Headers { return headers.With(rawResponseHeaderKey, "1") } diff --git a/transport/x/grpc/handler.go b/transport/x/grpc/handler.go index 58f8cc4fa..3a4c31c80 100644 --- a/transport/x/grpc/handler.go +++ b/transport/x/grpc/handler.go @@ -37,29 +37,32 @@ import ( ) type handler struct { - procedureServiceName string - serviceName string - methodName string - router transport.Router - onewayErrorHandler func(error) + yarpcServiceName string + grpcServiceName string + grpcMethodName string + router transport.Router + onewayErrorHandler func(error) } func newHandler( - procedureServiceName string, - serviceName string, - methodName string, + yarpcServiceName string, + grpcServiceName string, + grpcMethodName string, router transport.Router, onewayErrorHandler func(error), ) *handler { return &handler{ - procedureServiceName, - serviceName, - methodName, + yarpcServiceName, + grpcServiceName, + grpcMethodName, router, onewayErrorHandler, } } +// the grpc-go handler does not put the context.Context as the first argument +// so we must ignore this file for linting + func (h *handler) handle( server interface{}, ctx context.Context, @@ -76,7 +79,7 @@ func (h *handler) handle( transportRequest, &grpc.UnaryServerInfo{ noopGrpcStruct{}, - h.getFullMethod(), + toFullMethod(h.grpcServiceName, h.grpcMethodName), }, func(ctx context.Context, request interface{}) (interface{}, error) { transportRequest, ok := request.(*transport.Request) @@ -100,7 +103,7 @@ func (h *handler) getTransportRequest(ctx context.Context, decodeFunc func(inter return nil, err } if caller == "" { - caller = h.serviceName + caller = h.grpcServiceName } encoding, err := getEncoding(md) if err != nil { @@ -114,12 +117,17 @@ func (h *handler) getTransportRequest(ctx context.Context, decodeFunc func(inter return nil, err } if service == "" { - service = h.procedureServiceName + service = h.yarpcServiceName } headers, err := getApplicationHeaders(md) if err != nil { return nil, err } + // We must do this to indicate to the protobuf encoding that we + // need to return the raw response object over this transport. + // + // See the commentary within encoding/x/protobuf/inbound.go. + headers = protobuf.SetRawResponse(headers) var data []byte if err := decodeFunc(&data); err != nil { return nil, err @@ -128,7 +136,7 @@ func (h *handler) getTransportRequest(ctx context.Context, decodeFunc func(inter Caller: caller, Encoding: encoding, Service: service, - Procedure: procedureToName(h.serviceName, h.methodName), + Procedure: procedureToName(h.grpcServiceName, h.grpcMethodName), Headers: headers, Body: bytes.NewBuffer(data), } @@ -139,7 +147,7 @@ func (h *handler) getTransportRequest(ctx context.Context, decodeFunc func(inter } func (h *handler) getFullMethod() string { - return fmt.Sprintf("/%s/%s", h.serviceName, h.methodName) + return toFullMethod(h.grpcServiceName, h.grpcMethodName) } func (h *handler) call(ctx context.Context, transportRequest *transport.Request) (interface{}, error) { @@ -161,9 +169,9 @@ func (h *handler) callUnary(ctx context.Context, transportRequest *transport.Req if err := request.ValidateUnaryContext(ctx); err != nil { return nil, err } - protobuf.SetRawResponse(transportRequest.Headers) responseWriter := newResponseWriter() - // TODO: always return data? + // TODO: do we always want to return the data from responseWriter.Bytes, or return nil for the data if there is an error? + // For now, we are always returning the data err := transport.DispatchUnaryHandler(ctx, unaryHandler, time.Now(), transportRequest, responseWriter) err = multierr.Append(err, grpc.SendHeader(ctx, responseWriter.md)) data := responseWriter.Bytes() @@ -179,6 +187,7 @@ func (h *handler) callOneway(ctx context.Context, transportRequest *transport.Re // other transport implementation seem to create their own context for calls, need to understand better // This will not propagate opentracing, for example // Right now just letting context propagation test fail + // https://github.com/yarpc/yarpc-go/issues/904 h.onewayErrorHandler(transport.DispatchOnewayHandler(context.Background(), onewayHandler, transportRequest)) }() return nil diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index f2dcb070a..2a976a24d 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -128,6 +128,7 @@ func (i *Inbound) stop() error { func (i *Inbound) getServiceDescs() ([]*grpc.ServiceDesc, error) { // TODO: router.Procedures() is not guaranteed to be immutable + // https://github.com/yarpc/yarpc-go/issues/825 procedures := i.router.Procedures() if len(procedures) == 0 { return nil, errRouterHasNoProcedures From e0b13f4d7d74b6546e398546b5dbee16f95b7e5d Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 11 Apr 2017 15:57:07 +0200 Subject: [PATCH 78/82] Fix issues --- scripts/generate.sh | 1 + transport/x/grpc/handler.go | 44 ++++++++--------------------- transport/x/grpc/headers.go | 55 +++++++++++++++--------------------- transport/x/grpc/outbound.go | 22 +-------------- 4 files changed, 35 insertions(+), 87 deletions(-) diff --git a/scripts/generate.sh b/scripts/generate.sh index e6831cbcc..a04584d17 100755 --- a/scripts/generate.sh +++ b/scripts/generate.sh @@ -46,6 +46,7 @@ protoc_go_grpc() { protoc_yarpc_go() { protoc_with_imports "yarpc-go" "${1}" "" +} # Add "Generated by" header to Ragel-generated code. # diff --git a/transport/x/grpc/handler.go b/transport/x/grpc/handler.go index 3a4c31c80..a68b4e074 100644 --- a/transport/x/grpc/handler.go +++ b/transport/x/grpc/handler.go @@ -98,58 +98,36 @@ func (h *handler) getTransportRequest(ctx context.Context, decodeFunc func(inter if md == nil || !ok { return nil, fmt.Errorf("cannot get metadata from ctx: %v", ctx) } - caller, err := getCaller(md) - if err != nil { + transportRequest := &transport.Request{} + if err := populateTransportRequest(md, transportRequest); err != nil { return nil, err } - if caller == "" { - caller = h.grpcServiceName - } - encoding, err := getEncoding(md) - if err != nil { - return nil, err + if transportRequest.Caller == "" { + transportRequest.Caller = h.grpcServiceName } - if encoding == "" { - encoding = protobuf.Encoding - } - service, err := getService(md) - if err != nil { - return nil, err + if transportRequest.Encoding == "" { + transportRequest.Encoding = protobuf.Encoding } - if service == "" { - service = h.yarpcServiceName - } - headers, err := getApplicationHeaders(md) - if err != nil { - return nil, err + if transportRequest.Service == "" { + transportRequest.Service = h.yarpcServiceName } // We must do this to indicate to the protobuf encoding that we // need to return the raw response object over this transport. // // See the commentary within encoding/x/protobuf/inbound.go. - headers = protobuf.SetRawResponse(headers) + transportRequest.Headers = protobuf.SetRawResponse(transportRequest.Headers) var data []byte if err := decodeFunc(&data); err != nil { return nil, err } - transportRequest := &transport.Request{ - Caller: caller, - Encoding: encoding, - Service: service, - Procedure: procedureToName(h.grpcServiceName, h.grpcMethodName), - Headers: headers, - Body: bytes.NewBuffer(data), - } + transportRequest.Body = bytes.NewBuffer(data) + transportRequest.Procedure = procedureToName(h.grpcServiceName, h.grpcMethodName) if err := transport.ValidateRequest(transportRequest); err != nil { return nil, err } return transportRequest, nil } -func (h *handler) getFullMethod() string { - return toFullMethod(h.grpcServiceName, h.grpcMethodName) -} - func (h *handler) call(ctx context.Context, transportRequest *transport.Request) (interface{}, error) { handlerSpec, err := h.router.Choose(ctx, transportRequest) if err != nil { diff --git a/transport/x/grpc/headers.go b/transport/x/grpc/headers.go index 2603900b2..8a9e47756 100644 --- a/transport/x/grpc/headers.go +++ b/transport/x/grpc/headers.go @@ -26,6 +26,7 @@ import ( "go.uber.org/yarpc/api/transport" + "go.uber.org/multierr" "google.golang.org/grpc/metadata" ) @@ -36,23 +37,31 @@ const ( callerHeader = reservedHeaderPrefix + "caller" encodingHeader = reservedHeaderPrefix + "encoding" serviceHeader = reservedHeaderPrefix + "service" - procedureHeader = reservedHeaderPrefix + "procedure" ) -func addCaller(md metadata.MD, caller string) error { - return addToMetadata(md, callerHeader, caller) +// transportRequestToMetadata will populate all reserved and application headers +// from the Request into a new MD. +func transportRequestToMetadata(request *transport.Request) (metadata.MD, error) { + md := metadata.New(nil) + err := addToMetadata(md, callerHeader, request.Caller) + err = multierr.Append(err, addToMetadata(md, encodingHeader, string(request.Encoding))) + err = multierr.Append(err, addToMetadata(md, serviceHeader, request.Service)) + err = multierr.Append(err, addApplicationHeaders(md, request.Headers)) + return md, err } -func addEncoding(md metadata.MD, encoding transport.Encoding) error { - return addToMetadata(md, encodingHeader, string(encoding)) -} - -func addService(md metadata.MD, service string) error { - return addToMetadata(md, serviceHeader, service) -} - -func addProcedure(md metadata.MD, procedure string) error { - return addToMetadata(md, procedureHeader, procedure) +// populateTransportRequest will populate the Request with all reserved and application +// headers from the MD. +func populateTransportRequest(md metadata.MD, request *transport.Request) error { + caller, callerErr := getFromMetadata(md, callerHeader) + request.Caller = caller + encoding, encodingErr := getFromMetadata(md, encodingHeader) + request.Encoding = transport.Encoding(encoding) + service, serviceErr := getFromMetadata(md, serviceHeader) + request.Service = service + headers, headersErr := getApplicationHeaders(md) + request.Headers = headers + return multierr.Combine(callerErr, encodingErr, serviceErr, headersErr) } // add headers into md as application headers @@ -66,26 +75,6 @@ func addApplicationHeaders(md metadata.MD, headers transport.Headers) error { return nil } -func getCaller(md metadata.MD) (string, error) { - return getFromMetadata(md, callerHeader) -} - -func getEncoding(md metadata.MD) (transport.Encoding, error) { - encoding, err := getFromMetadata(md, encodingHeader) - if err != nil { - return "", err - } - return transport.Encoding(encoding), nil -} - -func getService(md metadata.MD) (string, error) { - return getFromMetadata(md, serviceHeader) -} - -func getProcedure(md metadata.MD) (string, error) { - return getFromMetadata(md, procedureHeader) -} - // get application headers from md // return error if any application error has more than one value func getApplicationHeaders(md metadata.MD) (transport.Headers, error) { diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 5b8eb73ff..69f53b892 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -116,7 +116,7 @@ func (o *Outbound) invoke( responseMD *metadata.MD, ) error { start := time.Now() - md, err := requestToMetadata(request) + md, err := transportRequestToMetadata(request) if err != nil { return err } @@ -174,26 +174,6 @@ func (o *Outbound) stop() error { return nil } -func requestToMetadata(request *transport.Request) (metadata.MD, error) { - md := metadata.New(nil) - if err := addCaller(md, request.Caller); err != nil { - return nil, err - } - if err := addEncoding(md, request.Encoding); err != nil { - return nil, err - } - if err := addService(md, request.Service); err != nil { - return nil, err - } - if err := addProcedure(md, request.Procedure); err != nil { - return nil, err - } - if err := addApplicationHeaders(md, request.Headers); err != nil { - return nil, err - } - return md, nil -} - func errorToGRPCError(ctx context.Context, request *transport.Request, start time.Time, err error) error { deadline, _ := ctx.Deadline() ttl := deadline.Sub(start) From 2cc24448e0431b002ebad7080495a7f6c0936751 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 11 Apr 2017 17:29:59 +0200 Subject: [PATCH 79/82] Fixes --- transport/x/grpc/handler.go | 6 +++++- transport/x/grpc/inbound.go | 10 +++++----- transport/x/grpc/inbound_test.go | 24 +++++++++++------------ transport/x/grpc/outbound.go | 9 ++++----- transport/x/grpc/util.go | 33 +++++++++++++++++--------------- transport/x/grpc/util_test.go | 20 +++++++++---------- 6 files changed, 53 insertions(+), 49 deletions(-) diff --git a/transport/x/grpc/handler.go b/transport/x/grpc/handler.go index a68b4e074..ae11fccfd 100644 --- a/transport/x/grpc/handler.go +++ b/transport/x/grpc/handler.go @@ -121,7 +121,11 @@ func (h *handler) getTransportRequest(ctx context.Context, decodeFunc func(inter return nil, err } transportRequest.Body = bytes.NewBuffer(data) - transportRequest.Procedure = procedureToName(h.grpcServiceName, h.grpcMethodName) + procedure, err := procedureToName(h.grpcServiceName, h.grpcMethodName) + if err != nil { + return nil, err + } + transportRequest.Procedure = procedure if err := transport.ValidateRequest(transportRequest); err != nil { return nil, err } diff --git a/transport/x/grpc/inbound.go b/transport/x/grpc/inbound.go index 2a976a24d..53091059b 100644 --- a/transport/x/grpc/inbound.go +++ b/transport/x/grpc/inbound.go @@ -133,24 +133,24 @@ func (i *Inbound) getServiceDescs() ([]*grpc.ServiceDesc, error) { if len(procedures) == 0 { return nil, errRouterHasNoProcedures } - serviceNameToServiceDesc := make(map[string]*grpc.ServiceDesc) + grpcServiceNameToServiceDesc := make(map[string]*grpc.ServiceDesc) for _, procedure := range procedures { serviceName, methodDesc, err := i.getServiceNameAndMethodDesc(procedure) if err != nil { return nil, err } - serviceDesc, ok := serviceNameToServiceDesc[serviceName] + serviceDesc, ok := grpcServiceNameToServiceDesc[serviceName] if !ok { serviceDesc = &grpc.ServiceDesc{ ServiceName: serviceName, HandlerType: (*noopGrpcInterface)(nil), } - serviceNameToServiceDesc[serviceName] = serviceDesc + grpcServiceNameToServiceDesc[serviceName] = serviceDesc } serviceDesc.Methods = append(serviceDesc.Methods, methodDesc) } - serviceDescs := make([]*grpc.ServiceDesc, 0, len(serviceNameToServiceDesc)) - for _, serviceDesc := range serviceNameToServiceDesc { + serviceDescs := make([]*grpc.ServiceDesc, 0, len(grpcServiceNameToServiceDesc)) + for _, serviceDesc := range grpcServiceNameToServiceDesc { serviceDescs = append(serviceDescs, serviceDesc) } return serviceDescs, nil diff --git a/transport/x/grpc/inbound_test.go b/transport/x/grpc/inbound_test.go index f13458c6f..d0f0582b8 100644 --- a/transport/x/grpc/inbound_test.go +++ b/transport/x/grpc/inbound_test.go @@ -35,9 +35,9 @@ import ( func TestGetServiceDescs(t *testing.T) { t.Parallel() for _, tt := range []struct { - Name string - Procedures []transport.Procedure - ServiceDescs []*grpc.ServiceDesc + Name string + Procedures []transport.Procedure + ExpectedServiceDescs []*grpc.ServiceDesc }{ { Name: "Basic", @@ -47,7 +47,7 @@ func TestGetServiceDescs(t *testing.T) { Service: "Example", }, }, - ServiceDescs: []*grpc.ServiceDesc{ + ExpectedServiceDescs: []*grpc.ServiceDesc{ { ServiceName: "KeyValue", Methods: []grpc.MethodDesc{ @@ -70,7 +70,7 @@ func TestGetServiceDescs(t *testing.T) { Service: "Example", }, }, - ServiceDescs: []*grpc.ServiceDesc{ + ExpectedServiceDescs: []*grpc.ServiceDesc{ { ServiceName: "KeyValue", Methods: []grpc.MethodDesc{ @@ -96,7 +96,7 @@ func TestGetServiceDescs(t *testing.T) { Service: "Example", }, }, - ServiceDescs: []*grpc.ServiceDesc{ + ExpectedServiceDescs: []*grpc.ServiceDesc{ { ServiceName: "KeyValue", Methods: []grpc.MethodDesc{ @@ -116,19 +116,19 @@ func TestGetServiceDescs(t *testing.T) { }, }, { - Name: "Default Method Name", + Name: "Default Service Name", Procedures: []transport.Procedure{ { Name: "Foo", Service: "Example", }, }, - ServiceDescs: []*grpc.ServiceDesc{ + ExpectedServiceDescs: []*grpc.ServiceDesc{ { - ServiceName: "Foo", + ServiceName: defaultServiceName, Methods: []grpc.MethodDesc{ { - MethodName: defaultMethodName, + MethodName: "Foo", }, }, }, @@ -140,8 +140,8 @@ func TestGetServiceDescs(t *testing.T) { inbound.SetRouter(newTestTransportRouter(tt.Procedures)) serviceDescs, err := inbound.getServiceDescs() require.NoError(t, err) - testServiceDescServiceNamesCovered(t, serviceDescs, tt.ServiceDescs) - for _, expectedServiceDesc := range tt.ServiceDescs { + testServiceDescServiceNamesCovered(t, serviceDescs, tt.ExpectedServiceDescs) + for _, expectedServiceDesc := range tt.ExpectedServiceDescs { serviceDesc := testFindServiceDesc(t, serviceDescs, expectedServiceDesc.ServiceName) testServiceDescMethodNamesCovered(t, serviceDesc, expectedServiceDesc) } diff --git a/transport/x/grpc/outbound.go b/transport/x/grpc/outbound.go index 69f53b892..475a4ddcf 100644 --- a/transport/x/grpc/outbound.go +++ b/transport/x/grpc/outbound.go @@ -27,15 +27,14 @@ import ( "sync" "time" - "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" + "go.uber.org/yarpc/api/transport" + "go.uber.org/yarpc/internal/errors" + internalsync "go.uber.org/yarpc/internal/sync" + "github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" - - "go.uber.org/yarpc/api/transport" - "go.uber.org/yarpc/internal/errors" - internalsync "go.uber.org/yarpc/internal/sync" ) var _ transport.UnaryOutbound = (*Outbound)(nil) diff --git a/transport/x/grpc/util.go b/transport/x/grpc/util.go index 23e445640..1bcf148a6 100644 --- a/transport/x/grpc/util.go +++ b/transport/x/grpc/util.go @@ -23,24 +23,21 @@ package grpc import ( "fmt" "net/url" - "strings" "go.uber.org/yarpc/internal/procedure" ) -const defaultMethodName = "__default__" +const defaultServiceName = "__default__" func procedureNameToServiceNameMethodName(procedureName string) (string, string, error) { - // TODO: this is hacky - serviceName, methodName := procedure.FromName(strings.Replace(procedureName, "/", "___", -1)) + serviceName, methodName := procedure.FromName(procedureName) if serviceName == "" { return "", "", fmt.Errorf("invalid procedure name: %s", procedureName) } if methodName == "" { - methodName = defaultMethodName + methodName = serviceName + serviceName = defaultServiceName } - // TODO: do we really need to do url.QueryEscape? - // Are there consequences if there is a diff from the string and the url.QueryEscape string? return url.QueryEscape(serviceName), url.QueryEscape(methodName), nil } @@ -53,17 +50,23 @@ func procedureNameToFullMethod(procedureName string) (string, error) { } func toFullMethod(serviceName string, methodName string) string { - if methodName == "" { - methodName = defaultMethodName + if serviceName == "" { + serviceName = defaultServiceName } return fmt.Sprintf("/%s/%s", serviceName, methodName) } -func procedureToName(serviceName string, methodName string) string { - if methodName == defaultMethodName { - // TODO: this is hacky - return strings.Replace(serviceName, "___", "/", -1) +func procedureToName(serviceName string, methodName string) (string, error) { + serviceName, err := url.QueryUnescape(serviceName) + if err != nil { + return "", err + } + methodName, err = url.QueryUnescape(methodName) + if err != nil { + return "", err + } + if serviceName == defaultServiceName { + return methodName, nil } - // TODO: this is hacky - return strings.Replace(procedure.ToName(serviceName, methodName), "___", "/", -1) + return procedure.ToName(serviceName, methodName), nil } diff --git a/transport/x/grpc/util_test.go b/transport/x/grpc/util_test.go index 19f18415d..7acdc490f 100644 --- a/transport/x/grpc/util_test.go +++ b/transport/x/grpc/util_test.go @@ -43,15 +43,15 @@ func TestProcedureNameFunctions(t *testing.T) { }, { ProcedureName: "foo", - ServiceName: "foo", - MethodName: "__default__", - FullMethod: "/foo/__default__", + ServiceName: "__default__", + MethodName: "foo", + FullMethod: "/__default__/foo", }, { ProcedureName: "foo/bar", - ServiceName: "foo___bar", - MethodName: "__default__", - FullMethod: "/foo___bar/__default__", + ServiceName: "__default__", + MethodName: url.QueryEscape("foo/bar"), + FullMethod: "/__default__/" + url.QueryEscape("foo/bar"), }, } { t.Run(tt.ProcedureName, func(t *testing.T) { @@ -59,12 +59,10 @@ func TestProcedureNameFunctions(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tt.ServiceName, serviceName) assert.Equal(t, tt.MethodName, methodName) - assert.Equal(t, tt.ProcedureName, procedureToName(serviceName, methodName)) + procedureName, err := procedureToName(serviceName, methodName) + assert.NoError(t, err) + assert.Equal(t, tt.ProcedureName, procedureName) assert.Equal(t, tt.FullMethod, toFullMethod(serviceName, methodName)) }) } } - -func TestDefaultMethodNameIsSameQueryEscaped(t *testing.T) { - assert.Equal(t, defaultMethodName, url.QueryEscape(defaultMethodName)) -} From c3d649e0fd3d79b9fbf74bcff80dde4ea2003630 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 12 Apr 2017 20:57:51 +0200 Subject: [PATCH 80/82] Remove GRPC variable that controls GRPC inbound in crossdock --- docker-compose.yml | 1 - internal/crossdock/server/oneway/server.go | 18 +++++------------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 56061d457..52c4dfbc7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -81,7 +81,6 @@ services: environment: - REDIS=enabled - CHERAMI=enabled - - GRPC=enabled node: dns_search: . diff --git a/internal/crossdock/server/oneway/server.go b/internal/crossdock/server/oneway/server.go index 71291a14d..031e369d3 100644 --- a/internal/crossdock/server/oneway/server.go +++ b/internal/crossdock/server/oneway/server.go @@ -66,13 +66,11 @@ func Start() { inbounds = append(inbounds, cheramiInboud) } - if useGrpc() { - listener, err := net.Listen("tcp", ":8090") - if err != nil { - log.Printf("err init grpc inbound %v\n", err) - } else { - inbounds = append(inbounds, grpc.NewInbound(listener)) - } + listener, err := net.Listen("tcp", ":8090") + if err != nil { + log.Printf("err init grpc inbound %v\n", err) + } else { + inbounds = append(inbounds, grpc.NewInbound(listener)) } dispatcher = yarpc.NewDispatcher(yarpc.Config{ @@ -114,12 +112,6 @@ func useCherami() bool { return os.Getenv("CHERAMI") == "enabled" } -// useGrpc checks to see if a grpc server is expected to be -// available -func useGrpc() bool { - return os.Getenv("GRPC") == "enabled" -} - func initCheramiInbound() (*cherami.Inbound, error) { destination := `/test/dest` consumerGroup := `/test/dest_cg` From 9980462af2e246eece17ffa61afcc8f8a9f6bb20 Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Wed, 12 Apr 2017 20:59:57 +0200 Subject: [PATCH 81/82] Update FooTransport to FooForTransport in crossdock client --- internal/crossdock/client/echo/json.go | 6 +++--- internal/crossdock/client/echo/protobuf.go | 6 +++--- internal/crossdock/client/echo/raw.go | 6 +++--- internal/crossdock/client/echo/thrift.go | 6 +++--- internal/crossdock/client/grpc/grpc.go | 8 ++++---- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/internal/crossdock/client/echo/json.go b/internal/crossdock/client/echo/json.go index fa1aa0bad..ebb77d07c 100644 --- a/internal/crossdock/client/echo/json.go +++ b/internal/crossdock/client/echo/json.go @@ -38,11 +38,11 @@ type jsonEcho struct { // JSON implements the 'json' behavior. func JSON(t crossdock.T) { - JSONTransport(t, "") + JSONForTransport(t, "") } -// JSONTransport implements the 'json' behavior for the given transport or behavior transport. -func JSONTransport(t crossdock.T, transport string) { +// JSONForTransport implements the 'json' behavior for the given transport or behavior transport. +func JSONForTransport(t crossdock.T, transport string) { t = createEchoT("json", transport, t) fatals := crossdock.Fatals(t) diff --git a/internal/crossdock/client/echo/protobuf.go b/internal/crossdock/client/echo/protobuf.go index 57268fb11..9f422c945 100644 --- a/internal/crossdock/client/echo/protobuf.go +++ b/internal/crossdock/client/echo/protobuf.go @@ -33,11 +33,11 @@ import ( // Protobuf implements the 'protobuf' behavior. func Protobuf(t crossdock.T) { - ProtobufTransport(t, "") + ProtobufForTransport(t, "") } -// ProtobufTransport implements the 'protobuf' behavior for the given transport or behavior transport. -func ProtobufTransport(t crossdock.T, transport string) { +// ProtobufForTransport implements the 'protobuf' behavior for the given transport or behavior transport. +func ProtobufForTransport(t crossdock.T, transport string) { t = createEchoT("protobuf", transport, t) fatals := crossdock.Fatals(t) diff --git a/internal/crossdock/client/echo/raw.go b/internal/crossdock/client/echo/raw.go index a45fad125..73d3f51ad 100644 --- a/internal/crossdock/client/echo/raw.go +++ b/internal/crossdock/client/echo/raw.go @@ -34,11 +34,11 @@ import ( // Raw implements the 'raw' behavior. func Raw(t crossdock.T) { - RawTransport(t, "") + RawForTransport(t, "") } -// RawTransport implements the 'raw' behavior for the given transport or behavior transport. -func RawTransport(t crossdock.T, transport string) { +// RawForTransport implements the 'raw' behavior for the given transport or behavior transport. +func RawForTransport(t crossdock.T, transport string) { t = createEchoT("raw", transport, t) fatals := crossdock.Fatals(t) diff --git a/internal/crossdock/client/echo/thrift.go b/internal/crossdock/client/echo/thrift.go index caa58b183..3faf6c026 100644 --- a/internal/crossdock/client/echo/thrift.go +++ b/internal/crossdock/client/echo/thrift.go @@ -34,11 +34,11 @@ import ( // Thrift implements the 'thrift' behavior. func Thrift(t crossdock.T) { - ThriftTransport(t, "") + ThriftForTransport(t, "") } -// ThriftTransport implements the 'thrift' behavior for the given transport or behavior transport. -func ThriftTransport(t crossdock.T, transport string) { +// ThriftForTransport implements the 'thrift' behavior for the given transport or behavior transport. +func ThriftForTransport(t crossdock.T, transport string) { t = createEchoT("thrift", transport, t) fatals := crossdock.Fatals(t) diff --git a/internal/crossdock/client/grpc/grpc.go b/internal/crossdock/client/grpc/grpc.go index eb322e116..c2f7d2f40 100644 --- a/internal/crossdock/client/grpc/grpc.go +++ b/internal/crossdock/client/grpc/grpc.go @@ -28,10 +28,10 @@ import ( ) var encodingToRunFunc = map[string]func(crossdock.T, string){ - "raw": echo.RawTransport, - "json": echo.JSONTransport, - "thrift": echo.ThriftTransport, - "protobuf": echo.ProtobufTransport, + "raw": echo.RawForTransport, + "json": echo.JSONForTransport, + "thrift": echo.ThriftForTransport, + "protobuf": echo.ProtobufForTransport, } // Run starts the grpc behavior, testing grpc over encodings. From f3ffdefbf26411975934caac08cb3ba79c2e2d7a Mon Sep 17 00:00:00 2001 From: Peter Edge Date: Tue, 18 Apr 2017 11:34:38 -0700 Subject: [PATCH 82/82] Remove grpc log code --- transport/x/grpc/log.go | 55 ----------------------------------------- 1 file changed, 55 deletions(-) delete mode 100644 transport/x/grpc/log.go diff --git a/transport/x/grpc/log.go b/transport/x/grpc/log.go deleted file mode 100644 index 816b443a6..000000000 --- a/transport/x/grpc/log.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2017 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package grpc - -import ( - "go.uber.org/zap" - "google.golang.org/grpc/grpclog" -) - -// SetLogger will set the given logger to be the logger for grpclog. -func SetLogger(logger *zap.Logger) { - grpclog.SetLogger(newLogger(logger)) -} - -type logger struct { - *zap.SugaredLogger -} - -func newLogger(l *zap.Logger) *logger { - return &logger{l.Sugar()} -} - -func (l *logger) Print(args ...interface{}) { - l.SugaredLogger.Info(args...) -} - -func (l *logger) Printf(format string, args ...interface{}) { - l.SugaredLogger.Infof(format, args...) -} - -func (l *logger) Println(args ...interface{}) { - l.SugaredLogger.Info(args...) -} - -func (l *logger) Fatalln(args ...interface{}) { - l.SugaredLogger.Fatal(args...) -}