feat: support gRPCS

Fix #3344.

Signed-off-by: spacewander <[email protected]>

.gitignore

@@ -70,5 +70,6 @@ t/lib/dubbo-backend/dubbo-backend-provider/target/
 /conf/config-*.yaml
 !/conf/config-default.yaml
 /conf/debug-*.yaml
+/build-cache/
 # release tar package
 *.tgz

.travis/linux_openresty_common_runner.sh

@@ -74,20 +74,17 @@ do_install() {
     cp .travis/ASF* .travis/openwhisk-utilities/scancode/
 
     ls -l ./
-    if [ ! -f "build-cache/grpc_server_example" ]; then
-        wget https://github.com/iresty/grpc_server_example/releases/download/20200901/grpc_server_example-amd64.tar.gz
+    if [ ! -f "build-cache/grpc_server_example_20210122" ]; then
+        wget https://github.com/api7/grpc_server_example/releases/download/20210122/grpc_server_example-amd64.tar.gz
         tar -xvf grpc_server_example-amd64.tar.gz
         mv grpc_server_example build-cache/
-    fi
-
-    if [ ! -f "build-cache/proto/helloworld.proto" ]; then
-        if [ ! -f "grpc_server_example/main.go" ]; then
-            git clone https://github.com/iresty/grpc_server_example.git grpc_server_example
-        fi
 
-        cd grpc_server_example/
+        git clone --depth 1 https://github.com/api7/grpc_server_example.git grpc_server_example
+        pushd grpc_server_example/ || exit 1
         mv proto/ ../build-cache/
-        cd ..
+        popd || exit 1
+
+        touch build-cache/grpc_server_example_20210122
     fi
 
     if [ ! -f "build-cache/grpcurl" ]; then
@@ -101,7 +98,10 @@ script() {
     export_or_prefix
     openresty -V
 
-    ./build-cache/grpc_server_example &
+    ./build-cache/grpc_server_example \
+        -grpc-address :50051 -grpcs-address :50052 \
+        -crt ./t/certs/apisix.crt -key ./t/certs/apisix.key \
+        &
 
     ./bin/apisix help
     ./bin/apisix init

.travis/linux_tengine_runner.sh

@@ -234,12 +234,6 @@ do_install() {
     cp .travis/ASF* .travis/openwhisk-utilities/scancode/
 
     ls -l ./
-
-    if [ ! -f "build-cache/grpc_server_example" ]; then
-        wget https://github.com/iresty/grpc_server_example/releases/download/20200901/grpc_server_example-amd64.tar.gz
-        tar -xvf grpc_server_example-amd64.tar.gz
-        mv grpc_server_example build-cache/
-    fi
 }
 
 script() {

apisix/cli/ngx_tpl.lua

@@ -560,6 +560,29 @@ http {
                 apisix.grpc_access_phase()
             }
 
+            grpc_set_header   Content-Type application/grpc;
+            grpc_socket_keepalive on;
+            grpc_pass         $upstream_scheme://apisix_backend;
+
+            header_filter_by_lua_block {
+                apisix.http_header_filter_phase()
+            }
+
+            body_filter_by_lua_block {
+                apisix.http_body_filter_phase()
+            }
+
+            log_by_lua_block {
+                apisix.http_log_phase()
+            }
+        }
+
+        # hack for OpenResty before 1.17.8, which doesn't support variable inside grpc_pass
+        location \@1_15_grpc_pass {
+            access_by_lua_block {
+                apisix.grpc_access_phase()
+            }
+
             grpc_set_header   Content-Type application/grpc;
             grpc_socket_keepalive on;
             grpc_pass         grpc://apisix_backend;
@@ -577,6 +600,28 @@ http {
             }
         }
 
+        location \@1_15_grpcs_pass {
+            access_by_lua_block {
+                apisix.grpc_access_phase()
+            }
+
+            grpc_set_header   Content-Type application/grpc;
+            grpc_socket_keepalive on;
+            grpc_pass         grpcs://apisix_backend;
+
+            header_filter_by_lua_block {
+                apisix.http_header_filter_phase()
+            }
+
+            body_filter_by_lua_block {
+                apisix.http_body_filter_phase()
+            }
+
+            log_by_lua_block {
+                apisix.http_log_phase()
+            }
+        }
+
         {% if enabled_plugins["dubbo-proxy"] then %}
         location @dubbo_pass {
             access_by_lua_block {

apisix/init.lua

@@ -28,6 +28,7 @@ local upstream_util = require("apisix.utils.upstream")
 local ctxdump       = require("resty.ctxdump")
 local ipmatcher     = require("resty.ipmatcher")
 local ngx           = ngx
+local ngx_version   = ngx.config.nginx_version
 local get_method    = ngx.req.get_method
 local ngx_exit      = ngx.exit
 local math          = math
@@ -511,6 +512,10 @@ function _M.http_access_phase()
         run_plugin("access", plugins, api_ctx)
     end
 
+    if route.value.service_protocol == "grpc" then
+        api_ctx.upstream_scheme = "grpc"
+    end
+
     local code, err = set_upstream(route, api_ctx)
     if code then
         core.log.error("failed to set upstream: ", err)
@@ -519,8 +524,13 @@ function _M.http_access_phase()
 
     set_upstream_host(api_ctx)
 
-    if route.value.service_protocol == "grpc" then
+    local up_scheme = api_ctx.upstream_scheme
+    if up_scheme == "grpcs" or up_scheme == "grpc" then
         ngx_var.ctx_ref = ctxdump.stash_ngx_ctx()
+        if ngx_version < 1017008 then
+            return ngx.exec("@1_15_" .. up_scheme .. "_pass")
+        end
+
         return ngx.exec("@grpc_pass")
     end
 

apisix/plugins/proxy-rewrite.lua

@@ -124,7 +124,6 @@ end
 
 do
     local upstream_vars = {
-        scheme     = "upstream_scheme",
         host       = "upstream_host",
         upgrade    = "upstream_upgrade",
         connection = "upstream_connection",
@@ -140,6 +139,9 @@ function _M.rewrite(conf, ctx)
             ctx.var[upstream_vars[name]] = conf[name]
         end
     end
+    if conf["scheme"] then
+        ctx.upstream_scheme = conf["scheme"]
+    end
 
     local upstream_uri = ctx.var.uri
     if conf.uri ~= nil then

apisix/schema_def.lua

@@ -341,6 +341,10 @@ local upstream_schema = {
             description = "the key of chash for dynamic load balancing",
             type = "string",
         },
+        scheme = {
+            default = "http",
+            enum = {"grpc", "grpcs", "http"}
+        },
         labels = {
             description = "key/value pairs to specify attributes",
             type = "object",

apisix/upstream.lua

@@ -22,6 +22,7 @@ local error = error
 local tostring = tostring
 local ipairs = ipairs
 local pairs = pairs
+local is_http = ngx.config.subsystem == "http"
 local upstreams
 local healthcheck
 
@@ -122,6 +123,17 @@ local function fetch_healthchecker(upstream)
 end
 
 
+local function set_upstream_scheme(ctx, upstream)
+    -- plugins like proxy-rewrite may already set ctx.upstream_scheme
+    if not ctx.upstream_scheme then
+        -- the old configuration doesn't have scheme field, so fallback to "http"
+        ctx.upstream_scheme = upstream.scheme or "http"
+    end
+
+    ctx.var["upstream_scheme"] = ctx.upstream_scheme
+end
+
+
 function _M.set_by_route(route, api_ctx)
     if api_ctx.upstream_conf then
         core.log.warn("upstream node has been specified, ",
@@ -175,11 +187,16 @@ function _M.set_by_route(route, api_ctx)
         return 502, "no valid upstream node"
     end
 
+    if not is_http then
+        return
+    end
+
     if nodes_count > 1 then
         local checker = fetch_healthchecker(up_conf)
         api_ctx.up_checker = checker
     end
 
+    set_upstream_scheme(api_ctx, up_conf)
     return
 end
 

doc/admin-api.md

@@ -73,7 +73,6 @@
 |upstream |False |Upstream|Enabled Upstream configuration, see [Upstream](architecture-design.md#upstream) for more||
 |upstream_id|False |Upstream|Enabled upstream id, see [Upstream](architecture-design.md#upstream) for more ||
 |service_id|False |Service|Binded Service configuration, see [Service](architecture-design.md#service) for more ||
-|service_protocol|False|Upstream protocol type|only `grpc`|Must set `grpc` if using `gRPC proxy` or `gRPC transcode`. |
 |labels   |False |Match Rules|Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}|
 |enable_websocket|False|Auxiliary| enable `websocket`(boolean), default `false`.||
 |status          |False|Auxiliary| enable this route, default `1`.|`1` to enable, `0` to disable|
@@ -525,6 +524,7 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst
 |desc     |optional|upstream usage scenarios, and more.|
 |pass_host            |optional|`pass` pass the client request host, `node` not pass the client request host, using the upstream node host, `rewrite` rewrite host by the configured `upstream_host`.|
 |upstream_host    |optional|This option is only valid if the `pass_host` is `rewrite`.|
+|scheme|optional |The scheme used when talk with the upstream. The value is one of ['http', 'grpc', 'grpcs'], default to 'http'.|
 |labels|optional |Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}|
 |create_time|optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|
 |update_time|optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|

doc/grpc-proxy.md

@@ -22,20 +22,20 @@
 # grpc-proxy
 
 proxying gRPC traffic:
-gRPC client -> APISIX -> gRPC server
+gRPC client -> APISIX -> gRPC/gRPCS server
 
 ## Parameters
 
-* `service_protocol`:  the route's option `service_protocol` must be `grpc`
-* `uri`:   format likes /service/method, Example：/helloworld.Greeter/SayHello
+* `scheme`: the `scheme` of the route's upstream must be `grpc` or `grpcs`.
+* `uri`: format likes /service/method, Example：/helloworld.Greeter/SayHello
 
 ### Example
 
 #### create proxying gRPC route
 
 Here's an example, to proxying gRPC service by specified route:
 
-* attention: the route's option `service_protocol` must be `grpc`
+* attention: the `scheme` of the route's upstream must be `grpc` or `grpcs`.
 * attention: APISIX use TLS‑encrypted HTTP/2 to expose gRPC service, so need to [config SSL certificate](https.md)
 * the grpc server example：[grpc_server_example](https://github.com/iresty/grpc_server_example)
 
@@ -44,8 +44,8 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
 {
     "methods": ["POST", "GET"],
     "uri": "/helloworld.Greeter/SayHello",
-    "service_protocol": "grpc",
     "upstream": {
+        "scheme": "grpc",
         "type": "roundrobin",
         "nodes": {
             "127.0.0.1:50051": 1
@@ -66,3 +66,22 @@ $ grpcurl -insecure -import-path /pathtoprotos  -proto helloworld.proto  -d '{"n
 ```
 
 This means that the proxying is working.
+
+### gRPCS
+
+If your gRPC service encrypts with TLS by itself (so called `gPRCS`, gPRC + TLS), you need to change the `scheme` to `grpcs`. The example above runs gRPCS service on port 50052, to proxy gRPC request, we need to use the configuration below:
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "methods": ["POST", "GET"],
+    "uri": "/helloworld.Greeter/SayHello",
+    "upstream": {
+        "scheme": "grpcs",
+        "type": "roundrobin",
+        "nodes": {
+            "127.0.0.1:50052": 1
+        }
+    }
+}'
+```

doc/plugins/grpc-transcode.md

@@ -64,15 +64,14 @@ curl http://127.0.0.1:9080/apisix/admin/proto/1 -H 'X-API-KEY: edd1c9f034335f136
 
 Here's an example, to enable the grpc-transcode plugin to specified route:
 
-* attention: the route's option `service_protocol` must be `grpc`
+* attention: the `scheme` in the route's upstream must be `grpc`
 * the grpc server example：[grpc_server_example](https://github.com/iresty/grpc_server_example)
 
 ```shell
 curl http://127.0.0.1:9080/apisix/admin/routes/111 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
 {
     "methods": ["GET"],
     "uri": "/grpctest",
-    "service_protocol": "grpc",
     "plugins": {
         "grpc-transcode": {
             "proto_id": "1",
@@ -81,6 +80,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/111 -H 'X-API-KEY: edd1c9f034335f
         }
     },
     "upstream": {
+        "scheme": "grpc",
         "type": "roundrobin",
         "nodes": {
             "127.0.0.1:50051": 1
@@ -141,7 +141,6 @@ curl http://127.0.0.1:9080/apisix/admin/routes/23 -H 'X-API-KEY: edd1c9f034335f1
 {
     "methods": ["GET"],
     "uri": "/zeebe/WorkflowInstanceCreate",
-    "service_protocol": "grpc",
     "plugins": {
         "grpc-transcode": {
             "proto_id": "1",
@@ -151,6 +150,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/23 -H 'X-API-KEY: edd1c9f034335f1
         }
     },
     "upstream": {
+        "scheme": "grpc",
         "type": "roundrobin",
         "nodes": {
             "127.0.0.1:26500": 1
@@ -192,6 +192,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/111 -H 'X-API-KEY: edd1c9f034335f
     "uri": "/grpctest",
     "plugins": {},
     "upstream": {
+        "scheme": "grpc",
         "type": "roundrobin",
         "nodes": {
             "127.0.0.1:50051": 1

doc/zh-cn/admin-api.md

@@ -64,7 +64,6 @@
 |upstream |`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 Upstream 配置，详见 [Upstream](architecture-design.md#upstream)||
 |upstream_id|`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Upstream|启用的 upstream id，详见 [Upstream](architecture-design.md#upstream)||
 |service_id|`plugins`、`script`、`upstream`/`upstream_id`、`service_id`至少选择一个 |Service|绑定的 Service 配置，详见 [Service](architecture-design.md#service)||
-|service_protocol|可选|上游协议类型|只可以是 "grpc"|启用 `gRPC proxy` 或 `gRPC transcode` 插件时，必须用 "grpc"|
 |name     |可选 |辅助   |标识路由名称|route-xxxx|
 |desc     |可选 |辅助   |标识描述、使用场景等。|客户 xxxx|
 |host     |可选 |匹配规则|当前请求域名，比如 `foo.com`；也支持泛域名，比如 `*.foo.com`。|"foo.com"|
@@ -534,6 +533,7 @@ APISIX 的 Upstream 除了基本的复杂均衡算法选择外，还支持对上
 |desc     |可选 |辅助|上游服务描述、使用场景等。||
 |pass_host            |可选|枚举|`pass` 透传客户端请求的 host, `node` 不透传客户端请求的 host, 使用 upstream node 配置的 host, `rewrite` 使用 `upstream_host` 配置的值重写 host 。||
 |upstream_host    |可选|辅助|只在 `pass_host` 配置为 `rewrite` 时有效。||
+|scheme|可选 |辅助|跟上游通信时使用的 scheme。需要是 ['http', 'grpc', 'grpcs'] 其中的一个，默认是 'http'。|
 |labels   |可选 |匹配规则|标识附加属性的键值对|{"version":"v2","build":"16","env":"production"}|
 |create_time|可选|辅助|单位为秒的 epoch 时间戳，如果不指定则自动创建|1602883670|
 |update_time|可选|辅助|单位为秒的 epoch 时间戳，如果不指定则自动创建|1602883670|