diff --git a/gateway/src/apicast/http_proxy.lua b/gateway/src/apicast/http_proxy.lua index 4293135c5..abbd9c634 100644 --- a/gateway/src/apicast/http_proxy.lua +++ b/gateway/src/apicast/http_proxy.lua @@ -1,5 +1,9 @@ local format = string.format local tostring = tostring +local ngx_flush = ngx.flush +local ngx_get_method = ngx.req.get_method +local ngx_http_version = ngx.req.http_version +local ngx_send_headers = ngx.send_headers local resty_url = require "resty.url" local resty_resolver = require 'resty.resolver' @@ -7,10 +11,20 @@ local round_robin = require 'resty.balancer.round_robin' local http_proxy = require 'resty.http.proxy' local file_reader = require("resty.file").file_reader local file_size = require("resty.file").file_size +local client_body_reader = require("resty.http.request_reader").get_client_body_reader +local send_response = require("resty.http.response_writer").send_response local concat = table.concat local _M = { } +local http_methods_with_body = { + POST = true, + PUT = true, + PATCH = true +} + +local DEFAULT_CHUNKSIZE = 32 * 1024 + function _M.reset() _M.balancer = round_robin.new() _M.resolver = resty_resolver @@ -84,52 +98,105 @@ local function absolute_url(uri) ) end -local function forward_https_request(proxy_uri, proxy_auth, uri, skip_https_connect) - -- This is needed to call ngx.req.get_body_data() below. - ngx.req.read_body() - - -- We cannot use resty.http's .get_client_body_reader(). - -- In POST requests with HTTPS, the result of that call is nil, and it - -- results in a time-out. - -- - -- - -- If ngx.req.get_body_data is nil, can be that the body is too big to - -- read and need to be cached in a local file. This request will return - -- nil, so after this we need to read the temp file. - -- https://github.com/openresty/lua-nginx-module#ngxreqget_body_data - local body = ngx.req.get_body_data() +local function handle_expect() + local expect = ngx.req.get_headers()["Expect"] + if type(expect) == "table" then + expect = expect[1] + end + + if expect and expect:lower() == "100-continue" then + ngx.status = 100 + local ok, err = ngx_send_headers() + + if not ok then + return nil, "failed to send response header: " .. (err or "unknown") + end + + ok, err = ngx_flush(true) + if not ok then + return nil, "failed to flush response header: " .. (err or "unknown") + end + end +end + +local function forward_https_request(proxy_uri, uri, proxy_opts) + local body, err + local sock + local opts = proxy_opts or {} + local req_method = ngx_get_method() local encoding = ngx.req.get_headers()["Transfer-Encoding"] + local is_chunked = encoding and encoding:lower() == "chunked" + + if http_methods_with_body[req_method] then + if opts.request_unbuffered and ngx_http_version() == 1.1 then + local _, err = handle_expect() + if err then + ngx.log(ngx.ERR, "failed to handle expect header, err: ", err) + return ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + end - if not body then - local temp_file_path = ngx.req.get_body_file() - ngx.log(ngx.INFO, "HTTPS Proxy: Request body is bigger than client_body_buffer_size, read the content from path='", temp_file_path, "'") - - if temp_file_path then - body, err = file_reader(temp_file_path) - if err then - ngx.log(ngx.ERR, "HTTPS proxy: Failed to read temp body file, err: ", err) - ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) - end - - if encoding == "chunked" then - -- If the body is smaller than "client_boby_buffer_size" the Content-Length header is - -- set based on the size of the buffer. However, when the body is rendered to a file, - -- we will need to calculate and manually set the Content-Length header based on the - -- file size - local contentLength, err = file_size(temp_file_path) - if err then - ngx.log(ngx.ERR, "HTTPS proxy: Failed to set content length, err: ", err) + if is_chunked then + -- The default ngx reader does not support chunked request + -- so we will need to get the raw request socket and manually + -- decode the chunked request + sock, err = ngx.req.socket(true) + else + sock, err = ngx.req.socket() + end + + if not sock then + ngx.log(ngx.ERR, "unable to obtain request socket: ", err) + return ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + end + + body = client_body_reader(sock, DEFAULT_CHUNKSIZE, is_chunked) + else + -- This is needed to call ngx.req.get_body_data() below. + ngx.req.read_body() + + -- We cannot use resty.http's .get_client_body_reader(). + -- In POST requests with HTTPS, the result of that call is nil, and it + -- results in a time-out. + -- + -- + -- If ngx.req.get_body_data is nil, can be that the body is too big to + -- read and need to be cached in a local file. This request will return + -- nil, so after this we need to read the temp file. + -- https://github.com/openresty/lua-nginx-module#ngxreqget_body_data + body = ngx.req.get_body_data() + + if not body then + local temp_file_path = ngx.req.get_body_file() + ngx.log(ngx.INFO, "HTTPS Proxy: Request body is bigger than client_body_buffer_size, read the content from path='", temp_file_path, "'") + + if temp_file_path then + body, err = file_reader(temp_file_path) + if err then + ngx.log(ngx.ERR, "HTTPS proxy: Failed to read temp body file, err: ", err) ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + end + + if is_chunked then + -- If the body is smaller than "client_boby_buffer_size" the Content-Length header is + -- set based on the size of the buffer. However, when the body is rendered to a file, + -- we will need to calculate and manually set the Content-Length header based on the + -- file size + local contentLength, err = file_size(temp_file_path) + if err then + ngx.log(ngx.ERR, "HTTPS proxy: Failed to set content length, err: ", err) + ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) + end + + ngx.req.set_header("Content-Length", tostring(contentLength)) + end end - - ngx.req.set_header("Content-Length", tostring(contentLength)) - end end - end - -- The whole request is buffered in the memory so remove the Transfer-Encoding: chunked - if ngx.var.http_transfer_encoding == "chunked" then - ngx.req.set_header("Transfer-Encoding", nil) + -- The whole request is buffered in the memory so remove the Transfer-Encoding: chunked + if is_chunked then + ngx.req.set_header("Transfer-Encoding", nil) + end + end end local request = { @@ -139,10 +206,10 @@ local function forward_https_request(proxy_uri, proxy_auth, uri, skip_https_conn path = format('%s%s%s', ngx.var.uri, ngx.var.is_args, ngx.var.query_string or ''), body = body, proxy_uri = proxy_uri, - proxy_auth = proxy_auth + proxy_auth = opts.proxy_auth } - local httpc, err = http_proxy.new(request, skip_https_connect) + local httpc, err = http_proxy.new(request, opts.skip_https_connect) if not httpc then ngx.log(ngx.ERR, 'could not connect to proxy: ', proxy_uri, ' err: ', err) @@ -154,8 +221,16 @@ local function forward_https_request(proxy_uri, proxy_auth, uri, skip_https_conn res, err = httpc:request(request) if res then - httpc:proxy_response(res) - httpc:set_keepalive() + if opts.request_unbuffered and is_chunked then + local bytes, err = send_response(sock, res, DEFAULT_CHUNKSIZE) + if not bytes then + ngx.log(ngx.ERR, "failed to send response: ", err) + return sock:send("HTTP/1.1 502 Bad Gateway") + end + else + httpc:proxy_response(res) + httpc:set_keepalive() + end else ngx.log(ngx.ERR, 'failed to proxy request to: ', proxy_uri, ' err : ', err) return ngx.exit(ngx.HTTP_BAD_GATEWAY) @@ -208,7 +283,13 @@ function _M.request(upstream, proxy_uri) return elseif uri.scheme == 'https' then upstream:rewrite_request() - forward_https_request(proxy_uri, proxy_auth, uri, upstream.skip_https_connect) + local proxy_opts = { + proxy_auth = proxy_auth, + skip_https_connect = upstream.skip_https_connect, + request_unbuffered = upstream.request_unbuffered + } + + forward_https_request(proxy_uri, uri, proxy_opts) return ngx.exit(ngx.OK) -- terminate phase else ngx.log(ngx.ERR, 'could not connect to proxy: ', proxy_uri, ' err: ', 'invalid request scheme') diff --git a/gateway/src/apicast/upstream.lua b/gateway/src/apicast/upstream.lua index 0aff47359..43c395c68 100644 --- a/gateway/src/apicast/upstream.lua +++ b/gateway/src/apicast/upstream.lua @@ -241,6 +241,7 @@ function _M:call(context) self:set_skip_https_connect_on_proxy(); end + self.request_unbuffered = context.request_unbuffered http_proxy.request(self, proxy_uri) else local err = self:rewrite_request() diff --git a/gateway/src/resty/http/request_reader.lua b/gateway/src/resty/http/request_reader.lua new file mode 100644 index 000000000..6b79e82d2 --- /dev/null +++ b/gateway/src/resty/http/request_reader.lua @@ -0,0 +1,47 @@ +local httpc = require "resty.resolver.http" + +local _M = { +} + +local cr_lf = "\r\n" + +-- chunked_reader return a body reader that translates the data read from +-- lua-resty-http client_body_reader to HTTP "chunked" format before returning it +-- +-- The chunked reader return nil when the final 0-length chunk is read +local function chunked_reader(sock, chunksize) + chunksize = chunksize or 65536 + local eof = false + local reader = httpc:get_client_body_reader(chunksize, sock) + if not reader then + return nil + end + + return function() + if eof then + return nil + end + + local buffer, err = reader() + if err then + return nil, err + end + if buffer then + local chunk = string.format("%x\r\n", #buffer) .. buffer .. cr_lf + return chunk + else + eof = true + return "0\r\n\r\n" + end + end +end + +function _M.get_client_body_reader(sock, chunksize, is_chunked) + if is_chunked then + return chunked_reader(sock, chunksize) + else + return httpc:get_client_body_reader(chunksize, sock) + end +end + +return _M diff --git a/gateway/src/resty/http/response_writer.lua b/gateway/src/resty/http/response_writer.lua new file mode 100644 index 000000000..9ab849676 --- /dev/null +++ b/gateway/src/resty/http/response_writer.lua @@ -0,0 +1,58 @@ +local _M = { +} + +local cr_lf = "\r\n" + +local function send(socket, data) + if not data or data == '' then + ngx.log(ngx.DEBUG, 'skipping sending nil') + return + end + + return socket:send(data) +end + +-- write_response writes response body reader to sock in the HTTP/1.x server response format, +-- The connection is closed if send() fails or when returning a non-zero +function _M.send_response(sock, response, chunksize) + local bytes, err + chunksize = chunksize or 65536 + + if not response then + ngx.log(ngx.ERR, "no response provided") + return + end + + if not sock then + return nil, "socket not initialized yet" + end + + -- Status line + local status = "HTTP/1.1 " .. response.status .. " " .. response.reason .. cr_lf + bytes, err = send(sock, status) + if not bytes then + return nil, "failed to send status line, err: " .. (err or "unknown") + end + + -- Write body + local reader = response.body_reader + repeat + local chunk, read_err + + chunk, read_err = reader(chunksize) + if read_err then + return nil, "failed to read response body, err: " .. (err or "unknown") + end + + if chunk then + bytes, err = send(sock, chunk) + if not bytes then + return nil, "failed to send response body, err: " .. (err or "unknown") + end + end + until not chunk + + return true, nil +end + +return _M diff --git a/t/apicast-policy-camel.t b/t/apicast-policy-camel.t index 2848b4ee8..6577e6391 100644 --- a/t/apicast-policy-camel.t +++ b/t/apicast-policy-camel.t @@ -3,6 +3,16 @@ use Test::APIcast::Blackbox 'no_plan'; require("http_proxy.pl"); +sub large_body { + my $res = ""; + for (my $i=0; $i <= 1024; $i++) { + $res = $res . "1111111 1111111 1111111 1111111\n"; + } + return $res; +} + +$ENV{'LARGE_BODY'} = large_body(); + repeat_each(1); run_tests(); @@ -742,3 +752,473 @@ EOF --- no_error_log [error] --- user_files fixture=tls.pl eval + + + +=== TEST 11: http_proxy with request_unbuffered policy +--- configuration +{ + "services": [ + { + "id": 42, + "backend_version": 1, + "backend_authentication_type": "service_token", + "backend_authentication_value": "token-value", + "proxy": { + "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast.policy.apicast" + }, + { + "name": "apicast.policy.camel", + "configuration": { + "http_proxy": "http://127.0.0.1:$TEST_NGINX_HTTP_PROXY_PORT" + } + } + ] + } + } + ] +} +--- backend + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream + server_name test-upstream.lvh.me; + location / { + echo_read_request_body; + echo_request_body; + } +--- request eval +"POST /?user_key= \n" . $ENV{LARGE_BODY} +--- response_body eval chomp +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: http://127.0.0.1:$TEST_NGINX_HTTP_PROXY_PORT +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file + + + +=== TEST 12: http_proxy with request_unbuffered policy and chunked body +--- configuration +{ + "services": [ + { + "id": 42, + "backend_version": 1, + "backend_authentication_type": "service_token", + "backend_authentication_value": "token-value", + "proxy": { + "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast.policy.apicast" + }, + { + "name": "apicast.policy.camel", + "configuration": { + "http_proxy": "http://127.0.0.1:$TEST_NGINX_HTTP_PROXY_PORT" + } + } + ] + } + } + ] +} +--- backend + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream + server_name test-upstream.lvh.me; + location / { + access_by_lua_block { + assert = require('luassert') + local content_length = ngx.req.get_headers()["Content-Length"] + local encoding = ngx.req.get_headers()["Transfer-Encoding"] + assert.equal('chunked', encoding) + assert.falsy(content_length) + } + echo_read_request_body; + echo_request_body; + } +--- more_headers +Transfer-Encoding: chunked +--- request eval +"POST /?user_key=value +". +sprintf("%x\r\n", length $ENV{"LARGE_BODY"}). +$ENV{LARGE_BODY} +."\r +0\r +\r +" +--- response_body eval +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: http://127.0.0.1:$TEST_NGINX_HTTP_PROXY_PORT +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file + + + +=== TEST 13: all_proxy with request_unbuffered policy +--- configuration +{ + "services": [ + { + "id": 42, + "backend_version": 1, + "backend_authentication_type": "service_token", + "backend_authentication_value": "token-value", + "proxy": { + "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast.policy.apicast" + }, + { + "name": "apicast.policy.camel", + "configuration": { + "all_proxy": "http://127.0.0.1:$TEST_NGINX_HTTP_PROXY_PORT" + } + } + ] + } + } + ] +} +--- backend + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream + server_name test-upstream.lvh.me; + location / { + echo_read_request_body; + echo_request_body; + } +--- request eval +"POST /?user_key= \n" . $ENV{LARGE_BODY} +--- response_body eval chomp +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: http://127.0.0.1:$TEST_NGINX_HTTP_PROXY_PORT +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file + + + +=== TEST 14: all_proxy with request_unbuffered policy and chunked body +--- configuration +{ + "services": [ + { + "id": 42, + "backend_version": 1, + "backend_authentication_type": "service_token", + "backend_authentication_value": "token-value", + "proxy": { + "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast.policy.apicast" + }, + { + "name": "apicast.policy.camel", + "configuration": { + "all_proxy": "http://127.0.0.1:$TEST_NGINX_HTTP_PROXY_PORT" + } + } + ] + } + } + ] +} +--- backend + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream + server_name test-upstream.lvh.me; + location / { + access_by_lua_block { + assert = require('luassert') + local content_length = ngx.req.get_headers()["Content-Length"] + local encoding = ngx.req.get_headers()["Transfer-Encoding"] + assert.equal('chunked', encoding) + assert.falsy(content_length) + } + echo_read_request_body; + echo_request_body; + } +--- more_headers +Transfer-Encoding: chunked +--- request eval +"POST /?user_key=value +". +sprintf("%x\r\n", length $ENV{"LARGE_BODY"}). +$ENV{LARGE_BODY} +."\r +0\r +\r +" +--- response_body eval +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: http://127.0.0.1:$TEST_NGINX_HTTP_PROXY_PORT +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file + + + +=== TEST 15: https_proxy with request_unbuffered policy, only upstream and proxy_pass will buffer +the request +--- init eval +$Test::Nginx::Util::PROXY_SSL_PORT = Test::APIcast::get_random_port(); +$Test::Nginx::Util::ENDPOINT_SSL_PORT = Test::APIcast::get_random_port(); +--- configuration random_port env eval +</tmp/out.txt' or die $!; +print $out $s; +close $out; +$s +--- response_body eval +$ENV{"LARGE_BODY"} +--- error_code: 200 +--- error_log env +using proxy: $TEST_NGINX_HTTP_PROXY +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file + + + +=== TEST 12: all_proxy with request_unbuffered policy +--- configuration +{ + "services": [ + { + "backend_version": 1, + "backend_authentication_type": "service_token", + "backend_authentication_value": "token-value", + "proxy": { + "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast.policy.apicast" + }, + { + "name": "apicast.policy.http_proxy", + "configuration": { + "all_proxy": "$TEST_NGINX_HTTP_PROXY" + } + } + ] + } + } + ] +} +--- backend +server_name test_backend.lvh.me; + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream +server_name test-upstream.lvh.me; + location /test { + echo_read_request_body; + echo_request_body; + } +--- request eval +"POST /test?user_key= \n" . $ENV{LARGE_BODY} +--- response_body eval chomp +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: $TEST_NGINX_HTTP_PROXY +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file + + + +=== TEST 13: all_proxy with request_unbuffered policy + chunked request +--- configuration +{ + "services": [ + { + "id": 42, + "backend_version": 1, + "backend_authentication_type": "service_token", + "backend_authentication_value": "token-value", + "proxy": { + "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast.policy.apicast" + }, + { + "name": "apicast.policy.http_proxy", + "configuration": { + "all_proxy": "$TEST_NGINX_HTTP_PROXY" + } + } + ] + } + } + ] +} +--- backend + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream +server_name test-upstream.lvh.me; + location / { + access_by_lua_block { + assert = require('luassert') + local content_length = ngx.req.get_headers()["Content-Length"] + local encoding = ngx.req.get_headers()["Transfer-Encoding"] + assert.equal('chunked', encoding) + assert.falsy(content_length) + } + echo_read_request_body; + echo_request_body; + } +--- more_headers +Transfer-Encoding: chunked +--- request eval +my $s = "POST /test?user_key=value +". +sprintf("%x\r\n", length $ENV{"LARGE_BODY"}). +$ENV{LARGE_BODY} +."\r +0\r +\r +"; +open my $out, '>/tmp/out.txt' or die $!; +print $out $s; +close $out; +$s +--- response_body eval +$ENV{"LARGE_BODY"} +--- error_code: 200 +--- error_log env +using proxy: $TEST_NGINX_HTTP_PROXY +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file + + + +=== TEST 14: https_proxy with request_unbuffered policy +--- configuration random_port env +{ + "services": [ + { + "backend_version": 1, + "proxy": { + "api_backend": "https://test-upstream.lvh.me:$TEST_NGINX_RANDOM_PORT", + "proxy_rules": [ + { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast.policy.apicast" + }, + { + "name": "apicast.policy.http_proxy", + "configuration": { + "https_proxy": "$TEST_NGINX_HTTPS_PROXY" + } + } + ] + } + } + ] +} +--- backend env + server_name test-backend.lvh.me; + listen $TEST_NGINX_RANDOM_PORT ssl; + ssl_certificate $TEST_NGINX_SERVER_ROOT/html/server.crt; + ssl_certificate_key $TEST_NGINX_SERVER_ROOT/html/server.key; + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream env +server_name test-upstream.lvh.me; +listen $TEST_NGINX_RANDOM_PORT ssl; +ssl_certificate $TEST_NGINX_SERVER_ROOT/html/server.crt; +ssl_certificate_key $TEST_NGINX_SERVER_ROOT/html/server.key; +location /test { + echo_read_request_body; + echo_request_body; +} +--- request eval +"POST /test?user_key= \n" . $ENV{LARGE_BODY} +--- response_body eval chomp +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: $TEST_NGINX_HTTPS_PROXY +proxy request: CONNECT test-upstream.lvh.me:$TEST_NGINX_RANDOM_PORT HTTP/1.1 +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file +--- user_files fixture=tls.pl eval + + + +=== TEST 15: https_proxy with request_unbuffered policy +--- configuration random_port env +{ + "services": [ + { + "backend_version": 1, + "proxy": { + "api_backend": "https://test-upstream.lvh.me:$TEST_NGINX_RANDOM_PORT", + "proxy_rules": [ + { "pattern": "/", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast.policy.apicast" + }, + { + "name": "apicast.policy.http_proxy", + "configuration": { + "https_proxy": "$TEST_NGINX_HTTPS_PROXY" + } + } + ] + } + } + ] +} +--- backend env + server_name test-backend.lvh.me; + listen $TEST_NGINX_RANDOM_PORT ssl; + ssl_certificate $TEST_NGINX_SERVER_ROOT/html/server.crt; + ssl_certificate_key $TEST_NGINX_SERVER_ROOT/html/server.key; + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream env +server_name test-upstream.lvh.me; +listen $TEST_NGINX_RANDOM_PORT ssl; +ssl_certificate $TEST_NGINX_SERVER_ROOT/html/server.crt; +ssl_certificate_key $TEST_NGINX_SERVER_ROOT/html/server.key; +location /test { + access_by_lua_block { + assert = require('luassert') + local content_length = ngx.req.get_headers()["Content-Length"] + local encoding = ngx.req.get_headers()["Transfer-Encoding"] + assert.equal('chunked', encoding) + assert.falsy(content_length) + } + echo_read_request_body; + echo_request_body; +} +--- more_headers +Transfer-Encoding: chunked +--- request eval +"POST /test?user_key=value +". +sprintf("%x\r\n", length $ENV{"LARGE_BODY"}). +$ENV{LARGE_BODY} +."\r +0\r +\r +" +--- response_body eval +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: $TEST_NGINX_HTTPS_PROXY +proxy request: CONNECT test-upstream.lvh.me:$TEST_NGINX_RANDOM_PORT HTTP/1.1 +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file +--- user_files fixture=tls.pl eval diff --git a/t/http-proxy.t b/t/http-proxy.t index 2710512f6..de623f152 100644 --- a/t/http-proxy.t +++ b/t/http-proxy.t @@ -1795,3 +1795,291 @@ proxy request: CONNECT test-upstream.lvh.me:$TEST_NGINX_RANDOM_PORT HTTP/1.1 --- no_error_log [error] --- user_files fixture=tls.pl eval + + + +=== TEST 32: HTTP_PROXY with request_unbuffered policy, only upstream server will buffer the +request +--- env eval +( + "http_proxy" => $ENV{TEST_NGINX_HTTP_PROXY}, + 'BACKEND_ENDPOINT_OVERRIDE' => "http://test_backend.lvh.me:$ENV{TEST_NGINX_SERVER_PORT}" +) +--- configuration +{ + "services": [ + { + "backend_version": 1, + "proxy": { + "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT", + "proxy_rules": [ + { "pattern": "/test", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast", + "version": "builtin", + "configuration": {} + } + ] + } + } + ] +} +--- backend +server_name test_backend.lvh.me; + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream +server_name test-upstream.lvh.me; + location /test { + echo_read_request_body; + echo_request_body; + } +--- request eval +"POST /test?user_key= \n" . $ENV{LARGE_BODY} +--- response_body eval chomp +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: $TEST_NGINX_HTTP_PROXY +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file + + + +=== TEST 33: HTTP_PROXY with request_unbuffered policy and chunked body, only upstream server will buffer the +request +--- env eval +( + "http_proxy" => $ENV{TEST_NGINX_HTTP_PROXY}, + 'BACKEND_ENDPOINT_OVERRIDE' => "http://test_backend.lvh.me:$ENV{TEST_NGINX_SERVER_PORT}" +) +--- configuration +{ + "services": [ + { + "backend_version": 1, + "proxy": { + "api_backend": "http://test-upstream.lvh.me:$TEST_NGINX_SERVER_PORT", + "proxy_rules": [ + { "pattern": "/test", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast", + "version": "builtin", + "configuration": {} + } + ] + } + } + ] +} +--- backend +server_name test_backend.lvh.me; + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream +server_name test-upstream.lvh.me; + location /test { + access_by_lua_block { + assert = require('luassert') + local content_length = ngx.req.get_headers()["Content-Length"] + local encoding = ngx.req.get_headers()["Transfer-Encoding"] + assert.equal('chunked', encoding) + assert.falsy(content_length) + } + echo_read_request_body; + echo_request_body; + } +--- more_headers +Transfer-Encoding: chunked +--- request eval +"POST /test?user_key=value +". +sprintf("%x\r\n", length $ENV{"LARGE_BODY"}). +$ENV{LARGE_BODY} +."\r +0\r +\r +" +--- response_body eval +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: $TEST_NGINX_HTTP_PROXY +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file + + + +=== TEST 34: HTTPS_PROXY with request_unbuffered policy, only the upstream server will buffer +the request +--- env random_port eval +( + 'https_proxy' => $ENV{TEST_NGINX_HTTPS_PROXY}, + 'BACKEND_ENDPOINT_OVERRIDE' => "https://test-backend.lvh.me:$ENV{TEST_NGINX_RANDOM_PORT}" +) +--- configuration random_port env +{ + "services": [ + { + "backend_version": 1, + "proxy": { + "api_backend": "https://test-upstream.lvh.me:$TEST_NGINX_RANDOM_PORT", + "proxy_rules": [ + { "pattern": "/test", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast", + "version": "builtin", + "configuration": {} + } + ] + } + } + ] +} +--- backend env + server_name test-backend.lvh.me; + listen $TEST_NGINX_RANDOM_PORT ssl; + ssl_certificate $TEST_NGINX_SERVER_ROOT/html/server.crt; + ssl_certificate_key $TEST_NGINX_SERVER_ROOT/html/server.key; + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream env +server_name test-upstream.lvh.me; +listen $TEST_NGINX_RANDOM_PORT ssl; +ssl_certificate $TEST_NGINX_SERVER_ROOT/html/server.crt; +ssl_certificate_key $TEST_NGINX_SERVER_ROOT/html/server.key; +location /test { + echo_read_request_body; + echo_request_body; +} +--- request eval +"POST /test?user_key= \n" . $ENV{LARGE_BODY} +--- response_body eval chomp +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: $TEST_NGINX_HTTPS_PROXY +proxy request: CONNECT test-upstream.lvh.me:$TEST_NGINX_RANDOM_PORT HTTP/1.1 +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file +--- user_files fixture=tls.pl eval + + + +=== TEST 35: HTTPS_PROXY with request_unbuffered policy, only the upstream server will buffer +the request +--- env random_port eval +( + 'https_proxy' => $ENV{TEST_NGINX_HTTPS_PROXY}, + 'BACKEND_ENDPOINT_OVERRIDE' => "https://test-backend.lvh.me:$ENV{TEST_NGINX_RANDOM_PORT}" +) +--- configuration random_port env +{ + "services": [ + { + "backend_version": 1, + "proxy": { + "api_backend": "https://test-upstream.lvh.me:$TEST_NGINX_RANDOM_PORT", + "proxy_rules": [ + { "pattern": "/test", "http_method": "POST", "metric_system_name": "hits", "delta": 2 } + ], + "policy_chain": [ + { + "name": "request_unbuffered" + }, + { + "name": "apicast", + "version": "builtin", + "configuration": {} + } + ] + } + } + ] +} +--- backend env + server_name test-backend.lvh.me; + listen $TEST_NGINX_RANDOM_PORT ssl; + ssl_certificate $TEST_NGINX_SERVER_ROOT/html/server.crt; + ssl_certificate_key $TEST_NGINX_SERVER_ROOT/html/server.key; + location /transactions/authrep.xml { + content_by_lua_block { + ngx.exit(ngx.OK) + } + } +--- upstream env +server_name test-upstream.lvh.me; +listen $TEST_NGINX_RANDOM_PORT ssl; +ssl_certificate $TEST_NGINX_SERVER_ROOT/html/server.crt; +ssl_certificate_key $TEST_NGINX_SERVER_ROOT/html/server.key; +location /test { + access_by_lua_block { + assert = require('luassert') + local content_length = ngx.req.get_headers()["Content-Length"] + local encoding = ngx.req.get_headers()["Transfer-Encoding"] + assert.equal('chunked', encoding) + assert.falsy(content_length) + } + echo_read_request_body; + echo_request_body; +} +--- more_headers +Transfer-Encoding: chunked +--- request eval +"POST /test?user_key=value +". +sprintf("%x\r\n", length $ENV{"LARGE_BODY"}). +$ENV{LARGE_BODY} +."\r +0\r +\r +" +--- response_body eval +$ENV{LARGE_BODY} +--- error_code: 200 +--- error_log env +using proxy: $TEST_NGINX_HTTPS_PROXY +proxy request: CONNECT test-upstream.lvh.me:$TEST_NGINX_RANDOM_PORT HTTP/1.1 +--- no_error_log +[error] +--- grep_error_log eval +qr/a client request body is buffered to a temporary file/ +--- grep_error_log_out +a client request body is buffered to a temporary file +--- user_files fixture=tls.pl eval