From 25d7d36ad69fe6d35bce5fdf4d71794a0e66eba6 Mon Sep 17 00:00:00 2001 From: David Ortiz Date: Wed, 16 May 2018 17:04:30 +0200 Subject: [PATCH 1/7] Add TemplateString --- gateway/src/apicast/template_string.lua | 116 ++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 gateway/src/apicast/template_string.lua diff --git a/gateway/src/apicast/template_string.lua b/gateway/src/apicast/template_string.lua new file mode 100644 index 000000000..b3ab53d5b --- /dev/null +++ b/gateway/src/apicast/template_string.lua @@ -0,0 +1,116 @@ +local Liquid = require 'liquid' +local LiquidTemplate = Liquid.Template +local LiquidInterpreterContext = Liquid.InterpreterContext +local LiquidFilterSet = Liquid.FilterSet +local LiquidResourceLimit = Liquid.ResourceLimit +local ngx = ngx + +local setmetatable = setmetatable +local pairs = pairs +local format = string.format + +local LiquidTemplateString = {} +local liquid_template_string_mt = { __index = LiquidTemplateString } + +-- Expose only ngx.* functions that we think could be useful and that do not +-- have any side-effects. + +-- Table of ngx.* functions that we think can be useful for templates and do +-- not have any side-effects. +-- @field escape_uri https://github.com/openresty/lua-nginx-module#ngxescape_uri +-- @field unescape_uri https://github.com/openresty/lua-nginx-module#ngxunescape_uri +-- @field encode_base64 https://github.com/openresty/lua-nginx-module#ngxencode_base64 +-- @field decode_base64 https://github.com/openresty/lua-nginx-module#ngxdecode_base64 +-- @field crc32_short https://github.com/openresty/lua-nginx-module#ngxcrc32_short +-- @field crc32_long https://github.com/openresty/lua-nginx-module#ngxcrc32_long +-- @field hmac_sha1 https://github.com/openresty/lua-nginx-module#ngxhmac_sha1 +-- @field md5 https://github.com/openresty/lua-nginx-module#ngxmd5 +-- @field md5_bin https://github.com/openresty/lua-nginx-module#ngxmd5_bin +-- @field sha1_bin https://github.com/openresty/lua-nginx-module#ngxsha1_bin +-- @field quote_sql_str https://github.com/openresty/lua-nginx-module#ngxquote_sql_str +-- @field today https://github.com/openresty/lua-nginx-module#ngxtoday +-- @field time https://github.com/openresty/lua-nginx-module#ngxtime +-- @field now https://github.com/openresty/lua-nginx-module#ngxnow +-- @field localtime https://github.com/openresty/lua-nginx-module#ngxlocaltime +-- @field utctime https://github.com/openresty/lua-nginx-module#ngxutctime +-- @field cookie_time https://github.com/openresty/lua-nginx-module#ngxcookie_time +-- @field http_time https://github.com/openresty/lua-nginx-module#ngxhttp_time +-- @field parse_http_time https://github.com/openresty/lua-nginx-module#ngxparse_http_time +local liquid_filters = { + escape_uri = ngx.escape_uri, + unescape_uri = ngx.unescape_uri, + encode_base64 = ngx.encode_base64, + decode_base64 = ngx.decode_base64, + crc32_short = ngx.crc32_short, + crc32_long = ngx.crc32_long, + hmac_sha1 = ngx.hmac_sha1, + md5 = ngx.md5, + md5_bin = ngx.md5_bin, + sha1_bin = ngx.sha1_bin, + quote_sql_str = ngx.quote_sql_str, + today = ngx.today, + time = ngx.time, + now = ngx.now, + localtime = ngx.localtime, + utctime = ngx.utctime, + cookie_time = ngx.cookie_time, + http_time = ngx.http_time, + parse_http_time = ngx.parse_http_time +} + +local liquid_filter_set = LiquidFilterSet:new() +for name, func in pairs(liquid_filters) do + liquid_filter_set:add_filter(name, func) +end + +-- Set resource limits to avoid loops +local liquid_resource_limit = LiquidResourceLimit:new(nil, nil, 0) + +function LiquidTemplateString.new(string) + return setmetatable({ template = LiquidTemplate:parse(string) }, + liquid_template_string_mt) +end + +function LiquidTemplateString:render(context) + return self.template:render( + LiquidInterpreterContext:new(context), + liquid_filter_set, + liquid_resource_limit + ) +end + +local PlainTemplateString = {} +local plain_template_string_mt = { __index = PlainTemplateString } + +function PlainTemplateString.new(string) + return setmetatable({ string = string }, plain_template_string_mt) +end + +function PlainTemplateString:render() + return self.string +end + +local template = { + plain = PlainTemplateString, + liquid = LiquidTemplateString +} + +local _M = {} + +--- Initialize a template +-- Initialize a liquid or a plain text template according to the given type. +-- @tparam string value String to construct the template from +-- @tparam string type Render the template as this type. +-- Can be 'liquid' or 'plain' +-- @treturn a template string and nil, err when an invalid type is given +function _M.new(value, type) + local template_mod = template[type] + + if template_mod then + return template_mod.new(value) + else + return nil, format('Invalid type specified: %s', type) + end +end + +return _M From 49d168acc99b75bbe3ba17a9618bcb214f1c183e Mon Sep 17 00:00:00 2001 From: David Ortiz Date: Wed, 16 May 2018 17:04:42 +0200 Subject: [PATCH 2/7] spec: add tests for TemplateString --- spec/template_string_spec.lua | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 spec/template_string_spec.lua diff --git a/spec/template_string_spec.lua b/spec/template_string_spec.lua new file mode 100644 index 000000000..4fb305758 --- /dev/null +++ b/spec/template_string_spec.lua @@ -0,0 +1,26 @@ +local TemplateString = require 'apicast.template_string' + +describe('template string', function() + it('renders plain text', function() + local plain_text_template = TemplateString.new('{{ a_key }}', 'plain') + assert.equals('{{ a_key }}', plain_text_template:render()) + end) + + it('renders liquid', function() + local liquid_template = TemplateString.new('{{ a_key }}', 'liquid') + assert.equals('a_value', liquid_template:render({ a_key = 'a_value' })) + end) + + it('can apply liquid filters', function() + local liquid_template = TemplateString.new('{{ "something" | md5 }}', 'liquid') + assert.equals(ngx.md5('something'), liquid_template:render({})) + end) + + describe('.new', function() + it('returns nil and an error with invalid type', function() + local template, err = TemplateString.new('some_string', 'invalid_type') + assert.is_nil(template) + assert.is_not_nil(err) + end) + end) +end) From 428ea56dda30b9a159785e935e2b4f7d568f8f79 Mon Sep 17 00:00:00 2001 From: David Ortiz Date: Wed, 16 May 2018 15:24:02 +0200 Subject: [PATCH 3/7] policy/headers: add value_type (plain,liquid) in schema --- .../apicast/policy/headers/apicast-policy.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gateway/src/apicast/policy/headers/apicast-policy.json b/gateway/src/apicast/policy/headers/apicast-policy.json index bf79859c5..d3aeda369 100644 --- a/gateway/src/apicast/policy/headers/apicast-policy.json +++ b/gateway/src/apicast/policy/headers/apicast-policy.json @@ -42,6 +42,21 @@ "value": { "description": "Value that will be added, set or pushed in the header", "type": "string" + }, + "value_type": { + "description": "How to evaluate 'value'", + "type": "string", + "oneOf": [ + { + "enum": ["plain"], + "title": "Evaluate 'value' as plain text." + }, + { + "enum": ["liquid"], + "title": "Evaluate 'value' as liquid." + } + ], + "default": "plain" } }, "required": ["op", "header", "value"] From abadec9b49c5928e8b884b2211b46bcb4f851371 Mon Sep 17 00:00:00 2001 From: David Ortiz Date: Wed, 16 May 2018 15:24:28 +0200 Subject: [PATCH 4/7] policy/headers: evaluate header values as liquid when specified in the config --- .../src/apicast/policy/headers/headers.lua | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/gateway/src/apicast/policy/headers/headers.lua b/gateway/src/apicast/policy/headers/headers.lua index 3a8dceace..85097fe45 100644 --- a/gateway/src/apicast/policy/headers/headers.lua +++ b/gateway/src/apicast/policy/headers/headers.lua @@ -9,6 +9,10 @@ local ipairs = ipairs local type = type local insert = table.insert +local TemplateString = require 'apicast.template_string' + +local default_value_type = 'plain' + local policy = require('apicast.policy') local _M = policy.new('Headers policy') @@ -69,10 +73,12 @@ local command_functions = { } -- header_type can be 'request' or 'response'. -local function run_commands(commands, header_type, ...) +local function run_commands(context, commands, header_type, ...) for _, command in ipairs(commands) do local command_func = command_functions[header_type][command.op] - command_func(command.header, command.value, ...) + local value = command.template_string:render(context) + + command_func(command.header, value, ...) end end @@ -85,6 +91,13 @@ local function init_config(config) return res end +local function build_templates(commands) + for _, command in ipairs(commands) do + command.template_string = TemplateString.new( + command.value, command.value_type or default_value_type) + end +end + --- Initialize a Headers policy -- @tparam[opt] table config -- @field[opt] request Table with the operations to apply to the request headers @@ -93,6 +106,7 @@ end -- 1) op: can be 'add', 'set' or 'push'. -- 2) header -- 3) value +-- 4) value_type (can be 'liquid' or 'plain'). Defaults to 'plain'. -- The push operation: -- 1) When the header is not set, creates it with the given value. -- 2) When the header is set, it creates a new header with the same name and @@ -108,18 +122,23 @@ end function _M.new(config) local self = new(config) self.config = init_config(config) + + for _, commands in ipairs({ self.config.request, self.config.response } ) do + build_templates(commands) + end + return self end -function _M:rewrite() +function _M:rewrite(context) -- This is here to avoid calling ngx.req.get_headers() in every command -- applied to the request headers. local req_headers = ngx.req.get_headers() or {} - run_commands(self.config.request, 'request', req_headers) + run_commands(context, self.config.request, 'request', req_headers) end -function _M:header_filter() - run_commands(self.config.response, 'response') +function _M:header_filter(context) + run_commands(context, self.config.response, 'response') end return _M From e29309a69fab841761076f37f3b47ef4148c35e5 Mon Sep 17 00:00:00 2001 From: David Ortiz Date: Wed, 16 May 2018 15:24:53 +0200 Subject: [PATCH 5/7] spec/policy/headers: test that values can be evaluated as liquid --- spec/policy/headers/headers_spec.lua | 98 ++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/spec/policy/headers/headers_spec.lua b/spec/policy/headers/headers_spec.lua index e293fc43a..862a5f0cd 100644 --- a/spec/policy/headers/headers_spec.lua +++ b/spec/policy/headers/headers_spec.lua @@ -105,6 +105,55 @@ describe('Headers policy', function() assert.same({ '2', '3' }, ngx.req.get_headers()[header]) end) end) + + describe('when the type of the value is specified', function() + describe("and it is 'liquid'", function() + it('evaluates the value as liquid', function() + local context = { var_in_context = 'some_value' } + + local config = { + [request_headers] = { + { + op = 'push', + header = header, + value = '{{ var_in_context }}', + value_type = 'liquid' + } + } + } + + local headers_policy = HeadersPolicy.new(config) + + headers_policy:rewrite(context) + + assert.same({ context.var_in_context }, ngx.req.get_headers()[header]) + end) + end) + + describe("and it is 'plain'", function() + it('evaluates the value as plain text', function() + local context = { var_in_context = 'some_value' } + local value = '{{ var_in_context }}' + + local config = { + [request_headers] = { + { + op = 'push', + header = header, + value = value, + value_type = 'plain' + } + } + } + + local headers_policy = HeadersPolicy.new(config) + + headers_policy:rewrite(context) + + assert.same({ value }, ngx.req.get_headers()[header]) + end) + end) + end) end) describe('.header_filter', function() @@ -201,5 +250,54 @@ describe('Headers policy', function() assert.same({ '2', '3' }, ngx.header[header]) end) end) + + describe('when the type of the value is specified', function() + describe("and it is 'liquid'", function() + it('evaluates the value as liquid', function() + local context = { var_in_context = 'some_value' } + + local config = { + [response_headers] = { + { + op = 'push', + header = header, + value = '{{ var_in_context }}', + value_type = 'liquid' + } + } + } + + local headers_policy = HeadersPolicy.new(config) + + headers_policy:header_filter(context) + + assert.same({ context.var_in_context }, ngx.header[header]) + end) + end) + + describe("and it is 'plain'", function() + it('evaluates the value as plain text', function() + local context = { var_in_context = 'some_value' } + local value = '{{ var_in_context }}' + + local config = { + [response_headers] = { + { + op = 'push', + header = header, + value = value, + value_type = 'plain' + } + } + } + + local headers_policy = HeadersPolicy.new(config) + + headers_policy:header_filter(context) + + assert.same({ value }, ngx.header[header]) + end) + end) + end) end) end) From acea692b9537d2b024669e259fdac0b91c2423b7 Mon Sep 17 00:00:00 2001 From: David Ortiz Date: Wed, 16 May 2018 15:25:07 +0200 Subject: [PATCH 6/7] t/apicast-policy-headers: test that values can be evaluated as liquid --- t/apicast-policy-headers.t | 67 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/t/apicast-policy-headers.t b/t/apicast-policy-headers.t index dbb0a8acb..634417a3f 100644 --- a/t/apicast-policy-headers.t +++ b/t/apicast-policy-headers.t @@ -451,3 +451,70 @@ yay, api backend --- error_code: 200 --- no_error_log [error] + +=== TEST 8: config with liquid templating +Test that we can apply filters and also get values from the policies context +--- backend + location /transactions/authrep.xml { + content_by_lua_block { + local expected = "service_token=token-value&service_id=42&usage%5Bhits%5D=2&user_key=value" + require('luassert').same(ngx.decode_args(expected), ngx.req.get_uri_args(0)) + } + } +--- configuration +{ + "services": [ + { + "id": 42, + "backend_version": 1, + "backend_authentication_type": "service_token", + "backend_authentication_value": "token-value", + "proxy": { + "api_backend": "http://test:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { "pattern": "/", "http_method": "GET", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { "name": "apicast.policy.apicast" }, + { + "name": "apicast.policy.headers", + "configuration": + { + "request": + [ + { + "op": "set", + "header": "New-Header-1", + "value": "{{ 'something' | md5 }}", + "value_type": "liquid" + }, + { + "op": "set", + "header": "New-Header-2", + "value": "{{ service.id }}", + "value_type": "liquid" + } + ] + } + } + ] + } + } + ] +} +--- upstream + location / { + content_by_lua_block { + local assert = require('luassert') + assert.same(ngx.md5("something"), ngx.req.get_headers()['New-Header-1']) + assert.same('42', ngx.req.get_headers()['New-Header-2']) + ngx.say('yay, api backend'); + } + } +--- request +GET /?user_key=value +--- response_body +yay, api backend +--- error_code: 200 +--- no_error_log +[error] From 6ec43073c8f49e0de523f0c62f35e9a0f1521f09 Mon Sep 17 00:00:00 2001 From: David Ortiz Date: Thu, 17 May 2018 11:48:24 +0200 Subject: [PATCH 7/7] CHANGELOG: add entry for liquid support in headers policy config --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65374c989..284daa221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - OpenTracing support [PR #669](https://github.com/3scale/apicast/pull/669) - Generate new policy scaffold from the CLI [PR #682](https://github.com/3scale/apicast/pull/682) - 3scale batcher policy [PR #685](https://github.com/3scale/apicast/pull/685), [PR #710](https://github.com/3scale/apicast/pull/710) +- Liquid templating support in the headers policy configuration [PR #716](https://github.com/3scale/apicast/pull/716) ### Changed