From 9a149745fdc30d823fdfb6a1cf6b7bccaf867ae4 Mon Sep 17 00:00:00 2001 From: spacewander Date: Fri, 4 Dec 2020 14:27:10 +0800 Subject: [PATCH 1/4] feat: route accroding to the graphql attributes --- apisix/cli/env.lua | 1 + apisix/core/ctx.lua | 80 +++++++- conf/config-default.yaml | 4 + doc/router-radixtree.md | 55 ++++++ rockspec/apisix-master-0.rockspec | 1 + t/APISIX.pm | 12 +- t/lib/test_admin.lua | 5 +- t/plugin/openid-connect.t | 22 +-- t/router/graphql.t | 297 ++++++++++++++++++++++++++++++ 9 files changed, 457 insertions(+), 20 deletions(-) create mode 100644 t/router/graphql.t diff --git a/apisix/cli/env.lua b/apisix/cli/env.lua index 0fc45f0e02f6..2ca8c3eb1bbb 100644 --- a/apisix/cli/env.lua +++ b/apisix/cli/env.lua @@ -45,6 +45,7 @@ return function (apisix_home, pkg_cpath_org, pkg_path_org) .. apisix_home .. "/deps/lib/lua/5.1/?.so;" local pkg_path = apisix_home .. "/?/init.lua;" + .. apisix_home .. "/deps/share/lua/5.1/?/init.lua;" .. apisix_home .. "/deps/share/lua/5.1/?.lua;;" package.cpath = pkg_cpath .. package.cpath diff --git a/apisix/core/ctx.lua b/apisix/core/ctx.lua index 840c0e314daf..a0c92799c42f 100644 --- a/apisix/core/ctx.lua +++ b/apisix/core/ctx.lua @@ -15,22 +15,96 @@ -- limitations under the License. -- local core_str = require("apisix.core.string") +local core_tab = require("apisix.core.table") +local request = require("apisix.core.request") local log = require("apisix.core.log") +local config_local = require("apisix.core.config_local") local tablepool = require("tablepool") local get_var = require("resty.ngxvar").fetch local get_request = require("resty.ngxvar").request local ck = require "resty.cookie" +local gq_parse = require("graphql").parse local setmetatable = setmetatable local sub_str = string.sub local rawset = rawset +local ngx = ngx local ngx_var = ngx.var local re_gsub = ngx.re.gsub +local ipairs = ipairs local type = type local error = error -local ngx = ngx +local pcall = pcall local _M = {version = 0.2} +local GRAPHQL_DEFAULT_MAX_SIZE = 1048576 -- 1MiB + + +local function parse_graphql(ctx) + local local_conf, err = config_local.local_conf() + if not local_conf then + return nil, "failed to get local conf: " .. err + end + + local max_size = GRAPHQL_DEFAULT_MAX_SIZE + local size = core_tab.try_read_attr(local_conf, "graphql", "max_size") + if size then + max_size = size + end + + local body, err = request.get_body(max_size, ctx) + if not body then + return nil, "failed to read graphql body: " .. err + end + + local ok, res = pcall(gq_parse, body) + if not ok then + return nil, "failed to parse graphql: " .. res .. " body: " .. body + end + + if #res.definitions == 0 then + return nil, "empty graphql: " .. body + end + + return res, nil +end + + +local function get_parsed_graphql(ctx) + if not ctx._graphql then + local res, err = parse_graphql(ctx) + if not res then + log.error(err) + ctx._graphql = {} + + else + if #res.definitions > 1 then + log.warn("Mutliple operations are not supported.", + "Only the first one is handled") + end + + local def = res.definitions[1] + local fields = def.selectionSet.selections + local root_fields = core_tab.new(#fields, 0) + for i, f in ipairs(fields) do + root_fields[i] = f.name.value + end + + local name = "" + if def.name and def.name.value then + name = def.name.value + end + + ctx._graphql = { + name = name, + operation = def.operation, + root_fields = root_fields, + } + end + end + + return ctx._graphql +end do @@ -84,6 +158,10 @@ do key = re_gsub(key, "-", "_", "jo") val = get_var(key, t._request) + elseif core_str.has_prefix(key, "graphql_") then + key = sub_str(key, 9) + val = get_parsed_graphql(t)[key] + elseif key == "route_id" then val = ngx.ctx.api_ctx and ngx.ctx.api_ctx.route_id diff --git a/conf/config-default.yaml b/conf/config-default.yaml index a59590ec3899..ae9f117059ac 100644 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -117,6 +117,7 @@ apisix: key_encrypt_salt: "edd1c9f0985e76a2" # If not set, will save origin ssl key into etcd. # If set this, must be a string of length 16. And it will encrypt ssl key with AES-128-CBC # !!! So do not change it after saving your ssl, it can't decrypt the ssl keys have be saved if you change !! + nginx_config: # config for render the template to genarate nginx.conf error_log: "logs/error.log" error_log_level: "warn" # warn,error @@ -195,6 +196,9 @@ etcd: # send: 2000 # default 2000ms # read: 5000 # default 5000ms +graphql: + max_size: 1048576 # the maximum size limitation of graphql in bytes, defualt 1MiB + plugins: # plugin list (sorted in alphabetical order) - api-breaker - authz-keycloak diff --git a/doc/router-radixtree.md b/doc/router-radixtree.md index ac3b5ac83a50..aaeda5d1bffc 100644 --- a/doc/router-radixtree.md +++ b/doc/router-radixtree.md @@ -94,3 +94,58 @@ $ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f ``` This route will require the request header `host` equal `iresty.com`, request cookie key `_device_id` equal `a66f0cdc4ba2df8c096f74c9110163a9` etc. + +### How to filter route by graphql attributes + +APISIX supports filtering route by some attributes of graphql. Currently we support: + +* graphql_operation +* graphql_name +* graphql_root_fields + +For instance, with graphql like this: +```graphql +query getRepo { + owner { + name + } + repo { + created + } +} +``` + +* The `graphql_operation` is `query` +* The `graphql_name` is `getRepo`, +* The `graphql_root_fields` is `["owner", "repo"]` + +We can filter such route out with: +```shell +$ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -i -d ' +{ + "methods": ["POST"], + "uri": "/_graphql", + "vars": [ + ["graphql_operation", "==", "query"], + ["graphql_name", "==", "getRepo"], + ["graphql_root_fields", "has", "owner"] + ], + "upstream": { + "type": "roundrobin", + "nodes": { + "39.97.63.215:80": 1 + } + } +}' +``` + +To prevent spending too much time reading invalid graphql request body, we only read the first 1 MiB +data from the request body. This limitation is configured via: + +```yaml +graphql: + max_size: 1048576 + +``` + +If you need to pass a graphql body which is larger than the limitation, you can increase the value in `conf/config.yaml`. diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index 833b28812d86..7e97f5abb652 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -56,6 +56,7 @@ dependencies = { "dkjson = 2.5-2", "resty-redis-cluster = 1.02-4", "lua-resty-expr = 1.0.0", + "graphql = 0.0.2", } build = { diff --git a/t/APISIX.pm b/t/APISIX.pm index b3b7e60ac4a8..c81907c4f44b 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -135,6 +135,11 @@ add_block_preprocessor(sub { my ($block) = @_; my $wait_etcd_sync = $block->wait_etcd_sync // 0.1; + my $lua_deps_path = <<_EOC_; + lua_package_path "$apisix_home/?.lua;$apisix_home/?/init.lua;$apisix_home/deps/share/lua/5.1/?/init.lua;$apisix_home/deps/share/lua/5.1/?.lua;$apisix_home/apisix/?.lua;$apisix_home/t/?.lua;;"; + lua_package_cpath "$apisix_home/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;"; +_EOC_ + my $main_config = $block->main_config // <<_EOC_; worker_rlimit_core 500M; env ENABLE_ETCD_AUTH; @@ -150,9 +155,7 @@ _EOC_ my $stream_enable = $block->stream_enable; my $stream_config = $block->stream_config // <<_EOC_; - lua_package_path "$apisix_home/?.lua;$apisix_home/?/init.lua;$apisix_home/deps/share/lua/5.1/?.lua;$apisix_home/apisix/?.lua;$apisix_home/t/?.lua;;"; - lua_package_cpath "$apisix_home/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;"; - + $lua_deps_path lua_socket_log_errors off; lua_shared_dict lrucache-lock-stream 10m; @@ -232,8 +235,7 @@ _EOC_ my $http_config = $block->http_config // ''; $http_config .= <<_EOC_; - lua_package_path "$apisix_home/?.lua;$apisix_home/?/init.lua;$apisix_home/deps/share/lua/5.1/?.lua;$apisix_home/apisix/?.lua;$apisix_home/t/?.lua;;"; - lua_package_cpath "$apisix_home/?.so;$apisix_home/deps/lib/lua/5.1/?.so;$apisix_home/deps/lib64/lua/5.1/?.so;;"; + $lua_deps_path lua_shared_dict plugin-limit-req 10m; lua_shared_dict plugin-limit-count 10m; diff --git a/t/lib/test_admin.lua b/t/lib/test_admin.lua index 4c1c16fd1530..8742aac19b46 100644 --- a/t/lib/test_admin.lua +++ b/t/lib/test_admin.lua @@ -108,14 +108,15 @@ function _M.comp_tab(left_tab, right_tab) local err dir_names = {} + local _ if type(left_tab) == "string" then - left_tab, err = json.decode(left_tab) + left_tab, _, err = json.decode(left_tab) if not left_tab then return false, "failed to decode expected data: " .. err end end if type(right_tab) == "string" then - right_tab, err = json.decode(right_tab) + right_tab, _, err = json.decode(right_tab) if not right_tab then return false, "failed to decode expected data: " .. err end diff --git a/t/plugin/openid-connect.t b/t/plugin/openid-connect.t index ef8f8a99d3e8..b1113ae74945 100644 --- a/t/plugin/openid-connect.t +++ b/t/plugin/openid-connect.t @@ -283,8 +283,7 @@ WWW-Authenticate: Bearer realm=apisix local t = require("lib.test_admin").test local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, - [[{ - "plugins": { + [[{ "plugins": { "openid-connect": { "client_id": "kbyuFDidLLm280LIwVFiazOqjO3ty8KH", "client_secret": "60Op4HFM0I8ajz0WdiStAbziZ-VFQttXuxixHHs2R7r7-CW8GR79l-mmLqMhc-Sa", @@ -294,10 +293,10 @@ WWW-Authenticate: Bearer realm=apisix "timeout": 10, "bearer_only": true, "scope": "apisix", - "public_key": "-----BEGIN PUBLIC KEY----- -MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANW16kX5SMrMa2t7F2R1w6Bk/qpjS4QQ -hnrbED3Dpsl9JXAx90MYsIWp51hBxJSE/EPVK8WF/sjHK1xQbEuDfEECAwEAAQ== ------END PUBLIC KEY-----", + "public_key": "-----BEGIN PUBLIC KEY-----\n]] .. + [[MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANW16kX5SMrMa2t7F2R1w6Bk/qpjS4QQ\n]] .. + [[hnrbED3Dpsl9JXAx90MYsIWp51hBxJSE/EPVK8WF/sjHK1xQbEuDfEECAwEAAQ==\n]] .. + [[-----END PUBLIC KEY-----", "token_signing_alg_values_expected": "RS256" } }, @@ -309,8 +308,7 @@ hnrbED3Dpsl9JXAx90MYsIWp51hBxJSE/EPVK8WF/sjHK1xQbEuDfEECAwEAAQ== }, "uri": "/hello" }]], - [[{ - "node": { + [[{ "node": { "value": { "plugins": { "openid-connect": { @@ -322,10 +320,10 @@ hnrbED3Dpsl9JXAx90MYsIWp51hBxJSE/EPVK8WF/sjHK1xQbEuDfEECAwEAAQ== "timeout": 10000, "bearer_only": true, "scope": "apisix", - "public_key": "-----BEGIN PUBLIC KEY----- -MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANW16kX5SMrMa2t7F2R1w6Bk/qpjS4QQ -hnrbED3Dpsl9JXAx90MYsIWp51hBxJSE/EPVK8WF/sjHK1xQbEuDfEECAwEAAQ== ------END PUBLIC KEY-----", + "public_key": "-----BEGIN PUBLIC KEY-----\n]] .. + [[MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBANW16kX5SMrMa2t7F2R1w6Bk/qpjS4QQ\n]] .. + [[hnrbED3Dpsl9JXAx90MYsIWp51hBxJSE/EPVK8WF/sjHK1xQbEuDfEECAwEAAQ==\n]] .. + [[-----END PUBLIC KEY-----", "token_signing_alg_values_expected": "RS256" } }, diff --git a/t/router/graphql.t b/t/router/graphql.t new file mode 100644 index 000000000000..727f48b82c02 --- /dev/null +++ b/t/router/graphql.t @@ -0,0 +1,297 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if (!$block->error_log && !$block->no_error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: set route by name +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [=[{ + "methods": ["POST"], + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello", + "vars": [["graphql_name", "==", "repo"]] + }]=] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + +=== TEST 2: route by name +--- request +POST /hello +query repo { + owner { + name + } +} +--- response_body +hello world + + + +=== TEST 3: set route by operation+name +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [=[{ + "methods": ["POST"], + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello", + "vars": [ + ["graphql_operation", "==", "mutation"], + ["graphql_name", "==", "repo"] + ] + }]=] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 4: route by operation+name +--- request +POST /hello +mutation repo($ep: Episode!, $review: ReviewInput!) { + createReview(episode: $ep, review: $review) { + stars + commentary + } +} +--- response_body +hello world + + + +=== TEST 5: route by operation+name, miss +--- request +POST /hello +query repo { + owner { + name + } +} +--- error_code: 404 + + + +=== TEST 6: multiple operations +--- request +POST /hello +mutation repo($ep: Episode!, $review: ReviewInput!) { + createReview(episode: $ep, review: $review) { + stars + commentary + } +} +query repo { + owner { + name + } +} +--- response_body +hello world +--- error_log +Mutliple operations are not supported + + + +=== TEST 7: bad graphql +--- request +POST /hello +AA +--- error_code: 404 +--- error_log +failed to parse graphql: Syntax error near line 1 body: AA + + + +=== TEST 8: set anonymous operaion name +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [=[{ + "methods": ["POST"], + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello", + "vars": [ + ["graphql_operation", "==", "query"], + ["graphql_name", "==", ""] + ] + }]=] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 9: route by anonymous name +--- request +POST /hello +query { + owner { + name + } +} +--- response_body +hello world + + + +=== TEST 10: limit the max size +--- yaml_config +graphql: + max_size: 5 +--- request +POST /hello +query { + owner { + name + } +} +--- error_code: 404 +--- error_log +failed to read graphql body + + + +=== TEST 11: set graphql_root_fields +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [=[{ + "methods": ["POST"], + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello", + "vars": [ + ["graphql_operation", "==", "query"], + ["graphql_root_fields", "has", "owner"] + ] + }]=] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 12: single root field +--- request +POST /hello +query { + owner { + name + } +} +--- response_body +hello world + + + +=== TEST 13: multiple root fields +--- request +POST /hello +query { + repo { + stars + } + owner { + name + } +} +--- response_body +hello world From 6780ff03036dadfbfbb0006a858c141533d17e5b Mon Sep 17 00:00:00 2001 From: spacewander Date: Mon, 7 Dec 2020 21:08:46 +0800 Subject: [PATCH 2/4] test --- t/router/graphql.t | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/t/router/graphql.t b/t/router/graphql.t index 727f48b82c02..31b0afdcc98d 100644 --- a/t/router/graphql.t +++ b/t/router/graphql.t @@ -295,3 +295,15 @@ query { } --- response_body hello world + + + +=== TEST 14: root fields mismatch +--- request +POST /hello +query { + repo { + name + } +} +--- error_code: 404 From 67c0ede2a9d0ae414be6dcacccb4495e6da0bc22 Mon Sep 17 00:00:00 2001 From: spacewander Date: Tue, 8 Dec 2020 12:08:11 +0800 Subject: [PATCH 3/4] tweak Signed-off-by: spacewander --- apisix/core/ctx.lua | 45 +++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/apisix/core/ctx.lua b/apisix/core/ctx.lua index a0c92799c42f..cc4f904b953d 100644 --- a/apisix/core/ctx.lua +++ b/apisix/core/ctx.lua @@ -66,7 +66,7 @@ local function parse_graphql(ctx) return nil, "empty graphql: " .. body end - return res, nil + return res end @@ -76,31 +76,31 @@ local function get_parsed_graphql(ctx) if not res then log.error(err) ctx._graphql = {} + return ctx._graphql + end - else - if #res.definitions > 1 then - log.warn("Mutliple operations are not supported.", - "Only the first one is handled") - end - - local def = res.definitions[1] - local fields = def.selectionSet.selections - local root_fields = core_tab.new(#fields, 0) - for i, f in ipairs(fields) do - root_fields[i] = f.name.value - end + if #res.definitions > 1 then + log.warn("Mutliple operations are not supported.", + "Only the first one is handled") + end - local name = "" - if def.name and def.name.value then - name = def.name.value - end + local def = res.definitions[1] + local fields = def.selectionSet.selections + local root_fields = core_tab.new(#fields, 0) + for i, f in ipairs(fields) do + root_fields[i] = f.name.value + end - ctx._graphql = { - name = name, - operation = def.operation, - root_fields = root_fields, - } + local name = "" + if def.name and def.name.value then + name = def.name.value end + + ctx._graphql = { + name = name, + operation = def.operation, + root_fields = root_fields, + } end return ctx._graphql @@ -159,6 +159,7 @@ do val = get_var(key, t._request) elseif core_str.has_prefix(key, "graphql_") then + -- trim the "graphql_" prefix key = sub_str(key, 9) val = get_parsed_graphql(t)[key] From 0f44ca2225433ecaaf71795f42654ff418a6ed84 Mon Sep 17 00:00:00 2001 From: spacewander Date: Tue, 8 Dec 2020 17:47:07 +0800 Subject: [PATCH 4/4] style Signed-off-by: spacewander --- apisix/core/ctx.lua | 62 +++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/apisix/core/ctx.lua b/apisix/core/ctx.lua index cc4f904b953d..f517c956e4e2 100644 --- a/apisix/core/ctx.lua +++ b/apisix/core/ctx.lua @@ -71,38 +71,40 @@ end local function get_parsed_graphql(ctx) - if not ctx._graphql then - local res, err = parse_graphql(ctx) - if not res then - log.error(err) - ctx._graphql = {} - return ctx._graphql - end - - if #res.definitions > 1 then - log.warn("Mutliple operations are not supported.", - "Only the first one is handled") - end - - local def = res.definitions[1] - local fields = def.selectionSet.selections - local root_fields = core_tab.new(#fields, 0) - for i, f in ipairs(fields) do - root_fields[i] = f.name.value - end - - local name = "" - if def.name and def.name.value then - name = def.name.value - end - - ctx._graphql = { - name = name, - operation = def.operation, - root_fields = root_fields, - } + if ctx._graphql then + return ctx._graphql end + local res, err = parse_graphql(ctx) + if not res then + log.error(err) + ctx._graphql = {} + return ctx._graphql + end + + if #res.definitions > 1 then + log.warn("Mutliple operations are not supported.", + "Only the first one is handled") + end + + local def = res.definitions[1] + local fields = def.selectionSet.selections + local root_fields = core_tab.new(#fields, 0) + for i, f in ipairs(fields) do + root_fields[i] = f.name.value + end + + local name = "" + if def.name and def.name.value then + name = def.name.value + end + + ctx._graphql = { + name = name, + operation = def.operation, + root_fields = root_fields, + } + return ctx._graphql end