Skip to content

Commit

Permalink
gRPC gateway init (#7019)
Browse files Browse the repository at this point in the history
* WIP: grpc server setup

* add register grpc routes

* updated go mod

* updated grpc for all modules

* added pb file for grpc gateway

* udpated gw file

* added a test for grpc route

* fixed conflicts

* added grpc server

* grpc tests added

* cleanup

* Fix gateway forward issue

* Add godoc

* updated tests

* fix lint

* fix tests

* fix tests

* fix tests

* fixed test

* Add grpc headers

* Fix error handling

* Fix tests - hacky

* Fix lint

* remove debug logs

* Fix review comments

* move grpc tests into a separate file

* Fix protobuf version

* Update x/capability/module.go

* Fix godoc

* Fix review suggestions

* Fix codec

* Add query params test for gateway request

* Fix gofmt

Co-authored-by: anilCSE <[email protected]>
Co-authored-by: Federico Kunze <[email protected]>
  • Loading branch information
3 people authored Aug 25, 2020
1 parent 1f7a278 commit 78194e1
Show file tree
Hide file tree
Showing 26 changed files with 423 additions and 11 deletions.
14 changes: 13 additions & 1 deletion client/grpc_query.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,18 @@ func (ctx Context) Invoke(grpcCtx gocontext.Context, method string, args, reply
if err != nil {
return err
}

// parse height header
md, _ := metadata.FromOutgoingContext(grpcCtx)
if heights := md.Get(grpctypes.GRPCBlockHeightHeader); len(heights) > 0 {
height, err := strconv.ParseInt(heights[0], 10, 64)
if err != nil {
return err
}

ctx = ctx.WithHeight(height)
}

req := abci.RequestQuery{
Path: method,
Data: reqBz,
Expand All @@ -47,7 +59,7 @@ func (ctx Context) Invoke(grpcCtx gocontext.Context, method string, args, reply
// We then parse all the call options, if the call option is a
// HeaderCallOption, then we manually set the value of that header to the
// metadata.
md := metadata.Pairs(grpctypes.GRPCBlockHeightHeader, strconv.FormatInt(res.Height, 10))
md = metadata.Pairs(grpctypes.GRPCBlockHeightHeader, strconv.FormatInt(res.Height, 10))
for _, callOpt := range opts {
header, ok := callOpt.(grpc.HeaderCallOption)
if !ok {
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ require (
github.com/cosmos/iavl v0.15.0-rc2
github.com/cosmos/ledger-cosmos-go v0.11.1
github.com/enigmampc/btcutil v1.0.3-0.20200723161021-e2fb6adb2a25
github.com/gogo/gateway v1.1.0
github.com/gogo/protobuf v1.3.1
github.com/golang/mock v1.4.4
github.com/golang/protobuf v1.4.2
Expand Down
11 changes: 7 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,8 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
github.com/gogo/gateway v1.1.0 h1:u0SuhL9+Il+UbjM9VIE3ntfRujKbvVpFvNB4HbjeVQ0=
github.com/gogo/gateway v1.1.0/go.mod h1:S7rR8FRQyG3QFESeSv4l2WnsyzlCLG0CzBbUUo/mbic=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
Expand Down Expand Up @@ -213,6 +215,7 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
Expand All @@ -231,14 +234,10 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg=
github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/handlers v1.5.0 h1:4wjo3sf9azi99c8hTmyaxp9y5S+pFszsy3pP0rAw/lw=
github.com/gorilla/handlers v1.5.0/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
Expand All @@ -248,7 +247,9 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.14.7 h1:Nk5kuHrnWUTf/0GL1a/vchH/om9Ap2/HnVna+jYZgTY=
github.com/grpc-ecosystem/grpc-gateway v1.14.7/go.mod h1:oYZKL012gGh6LMyg/xA7Q2yq6j8bu0wa+9w14EEthWU=
Expand Down Expand Up @@ -740,6 +741,7 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaR
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
Expand All @@ -761,6 +763,7 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
Expand Down
49 changes: 46 additions & 3 deletions server/api/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,18 @@ import (
"strings"
"time"

"github.com/gogo/gateway"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
"github.com/grpc-ecosystem/grpc-gateway/runtime"
"github.com/rakyll/statik/fs"
"github.com/tendermint/tendermint/libs/log"
tmrpcserver "github.com/tendermint/tendermint/rpc/jsonrpc/server"

"github.com/cosmos/cosmos-sdk/client"
"github.com/cosmos/cosmos-sdk/server/config"
"github.com/cosmos/cosmos-sdk/telemetry"
grpctypes "github.com/cosmos/cosmos-sdk/types/grpc"
"github.com/cosmos/cosmos-sdk/types/rest"

// unnamed import of statik for swagger UI support
Expand All @@ -24,19 +27,54 @@ import (

// Server defines the server's API interface.
type Server struct {
Router *mux.Router
ClientCtx client.Context
Router *mux.Router
GRPCRouter *runtime.ServeMux
ClientCtx client.Context

logger log.Logger
metrics *telemetry.Metrics
listener net.Listener
}

// CustomGRPCHeaderMatcher for mapping request headers to
// GRPC metadata.
// HTTP headers that start with 'Grpc-Metadata-' are automatically mapped to
// gRPC metadata after removing prefix 'Grpc-Metadata-'. We can use this
// CustomGRPCHeaderMatcher if headers don't start with `Grpc-Metadata-`
func CustomGRPCHeaderMatcher(key string) (string, bool) {
switch strings.ToLower(key) {
case grpctypes.GRPCBlockHeightHeader:
return grpctypes.GRPCBlockHeightHeader, true
default:
return runtime.DefaultHeaderMatcher(key)
}
}

func New(clientCtx client.Context, logger log.Logger) *Server {
// The default JSON marshaller used by the gRPC-Gateway is unable to marshal non-nullable non-scalar fields.
// Using the gogo/gateway package with the gRPC-Gateway WithMarshaler option fixes the scalar field marshalling issue.
marshalerOption := &gateway.JSONPb{
EmitDefaults: true,
Indent: " ",
OrigName: true,
}

return &Server{
Router: mux.NewRouter(),
ClientCtx: clientCtx,
logger: logger,
GRPCRouter: runtime.NewServeMux(
// Custom marshaler option is required for gogo proto
runtime.WithMarshalerOption(runtime.MIMEWildcard, marshalerOption),

// This is necessary to get error details properly
// marshalled in unary requests.
runtime.WithProtoErrorHandler(runtime.DefaultHTTPProtoErrorHandler),

// Custom header matcher for mapping request headers to
// GRPC metadata
runtime.WithIncomingHeaderMatcher(CustomGRPCHeaderMatcher),
),
}
}

Expand Down Expand Up @@ -70,6 +108,7 @@ func (s *Server) Start(cfg config.Config) error {
return err
}

s.registerGRPCRoutes()
s.listener = listener
var h http.Handler = s.Router

Expand All @@ -93,7 +132,11 @@ func (s *Server) registerSwaggerUI() {
}

staticServer := http.FileServer(statikFS)
s.Router.PathPrefix("/").Handler(staticServer)
s.Router.PathPrefix("/legacy").Handler(staticServer)
}

func (s *Server) registerGRPCRoutes() {
s.Router.PathPrefix("/").Handler(s.GRPCRouter)
}

func (s *Server) registerMetrics() {
Expand Down
1 change: 0 additions & 1 deletion server/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,6 @@ func startInProcess(ctx *Context, legacyAminoCdc *codec.LegacyAmino, appCreator

apiSrv = api.New(clientCtx, ctx.Logger.With("module", "api-server"))
app.RegisterAPIRoutes(apiSrv)

errCh := make(chan error)

go func() {
Expand Down
1 change: 1 addition & 0 deletions simapp/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,7 @@ func (app *SimApp) RegisterAPIRoutes(apiSvr *api.Server) {
rpc.RegisterRoutes(clientCtx, apiSvr.Router)
authrest.RegisterTxRoutes(clientCtx, apiSvr.Router)
ModuleBasics.RegisterRESTRoutes(clientCtx, apiSvr.Router)
ModuleBasics.RegisterGRPCRoutes(apiSvr.ClientCtx, apiSvr.GRPCRouter)
}

// GetMaccPerms returns a copy of the module account permissions
Expand Down
37 changes: 37 additions & 0 deletions tests/mocks/types_module_module.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

38 changes: 38 additions & 0 deletions testutil/rest.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package testutil

import (
"io/ioutil"
"net/http"
)

// GetRequestWithHeaders defines a wrapper around an HTTP GET request with a provided URL
// and custom headers
// An error is returned if the request or reading the body fails.
func GetRequestWithHeaders(url string, headers map[string]string) ([]byte, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}

client := &http.Client{}

for key, value := range headers {
req.Header.Set(key, value)
}

res, err := client.Do(req)
if err != nil {
return nil, err
}

body, err := ioutil.ReadAll(res.Body)
if err != nil {
return nil, err
}

if err = res.Body.Close(); err != nil {
return nil, err
}

return body, nil
}
9 changes: 9 additions & 0 deletions types/module/module.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"encoding/json"

"github.com/gogo/protobuf/grpc"
"github.com/grpc-ecosystem/grpc-gateway/runtime"

"github.com/gorilla/mux"
"github.com/spf13/cobra"
Expand All @@ -56,6 +57,7 @@ type AppModuleBasic interface {

// client functionality
RegisterRESTRoutes(client.Context, *mux.Router)
RegisterGRPCRoutes(client.Context, *runtime.ServeMux)
GetTxCmd() *cobra.Command
GetQueryCmd() *cobra.Command
}
Expand Down Expand Up @@ -114,6 +116,13 @@ func (bm BasicManager) RegisterRESTRoutes(clientCtx client.Context, rtr *mux.Rou
}
}

// RegisterGRPCRoutes registers all module rest routes
func (bm BasicManager) RegisterGRPCRoutes(clientCtx client.Context, rtr *runtime.ServeMux) {
for _, b := range bm {
b.RegisterGRPCRoutes(clientCtx, rtr)
}
}

// AddTxCommands adds all tx commands to the rootTxCmd.
//
// TODO: Remove clientCtx argument.
Expand Down
5 changes: 5 additions & 0 deletions x/auth/module.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"math/rand"

"github.com/gogo/protobuf/grpc"
"github.com/grpc-ecosystem/grpc-gateway/runtime"

"github.com/gorilla/mux"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -64,6 +65,10 @@ func (AppModuleBasic) RegisterRESTRoutes(clientCtx client.Context, rtr *mux.Rout
rest.RegisterRoutes(clientCtx, rtr, types.StoreKey)
}

// RegisterGRPCRoutes registers the gRPC Gateway routes for the auth module.
func (a AppModuleBasic) RegisterGRPCRoutes(_ client.Context, _ *runtime.ServeMux) {
}

// GetTxCmd returns the root tx command for the auth module.
func (AppModuleBasic) GetTxCmd() *cobra.Command {
return cli.GetTxCmd()
Expand Down
Loading

0 comments on commit 78194e1

Please sign in to comment.