feat(clustering): hybrid forward proxy in https (#9773)

Add support of talk to an HTTP tunnel in https

FTI-2996

kong.conf.default

@@ -225,6 +225,12 @@
 #proxy_server =                  # Proxy server defined as a URL. Kong will only use this
                                  # option if any component is explictly configured
                                  # to use proxy.
+
+
+#proxy_server_ssl_verify = off   # Toggles server certificate verification if
+                                 # `proxy_server` is in HTTPS.
+                                 # See the `lua_ssl_trusted_certificate`
+                                 # setting to specify a certificate authority.
 #------------------------------------------------------------------------------
 # HYBRID MODE
 #------------------------------------------------------------------------------

kong/clustering/utils.lua

@@ -42,6 +42,10 @@ local OCSP_TIMEOUT = constants.CLUSTERING_OCSP_TIMEOUT
 
 local KONG_VERSION = kong.version
 
+
+local prefix = kong.configuration.prefix or require("pl.path").abspath(ngx.config.prefix())
+local CLUSTER_PROXY_SSL_TERMINATOR_SOCK = fmt("unix:%s/cluster_proxy_ssl_terminator.sock", prefix)
+
 local _M = {}
 
 
@@ -335,10 +339,17 @@ local function parse_proxy_url(conf)
   if proxy_server then
     -- assume proxy_server is validated in conf_loader
     local parsed = parse_url(proxy_server)
-    ret.proxy_url = fmt("%s://%s:%s", parsed.scheme, parsed.host, parsed.port or 443)
-    ret.scheme = parsed.scheme
-    ret.host = parsed.host
-    ret.port = parsed.port
+    if parsed.scheme == "https" then
+      ret.proxy_url = CLUSTER_PROXY_SSL_TERMINATOR_SOCK
+      -- hide other fields to avoid it being accidently used
+      -- the connection details is statically rendered in nginx template
+
+    else -- http
+      ret.proxy_url = fmt("%s://%s:%s", parsed.scheme, parsed.host, parsed.port or 443)
+      ret.scheme = parsed.scheme
+      ret.host = parsed.host
+      ret.port = parsed.port
+    end
 
     if parsed.user and parsed.password then
       ret.proxy_authorization = "Basic " .. encode_base64(parsed.user  .. ":" .. parsed.password)

kong/conf_loader/init.lua

@@ -543,6 +543,7 @@ local CONF_INFERENCES = {
   opentelemetry_tracing_sampling_rate = { typ = "number" },
 
   proxy_server = { typ = "string" },
+  proxy_server_ssl_verify = { typ = "boolean" },
 }
 
 
@@ -995,8 +996,8 @@ local function check_and_infer(conf, opts)
     elseif not parsed.scheme then
       errors[#errors + 1] = "proxy_server missing scheme"
 
-    elseif parsed.scheme ~= "http" then
-      errors[#errors + 1] = "proxy_server only supports \"http\", got " .. parsed.scheme
+    elseif parsed.scheme ~= "http" and parsed.scheme ~= "https" then
+      errors[#errors + 1] = "proxy_server only supports \"http\" and \"https\", got " .. parsed.scheme
 
     elseif not parsed.host then
       errors[#errors + 1] = "proxy_server missing host"
@@ -1861,6 +1862,15 @@ local function load(path, custom_conf, opts)
 
   log.verbose("prefix in use: %s", conf.prefix)
 
+  -- hybrid mode HTTP tunneling (CONNECT) proxy inside HTTPS
+  if conf.cluster_use_proxy then
+    -- throw err, assume it's already handled in check_and_infer
+    local parsed = assert(socket_url.parse(conf.proxy_server))
+    if parsed.scheme == "https" then
+      conf.cluster_ssl_tunnel = fmt("%s:%s", parsed.host, parsed.port or 443)
+    end
+  end
+
   -- initialize the dns client, so the globally patched tcp.connect method
   -- will work from here onwards.
   assert(require("kong.tools.dns")(conf))

kong/resty/websocket/client.lua

@@ -21,6 +21,7 @@ local encode_base64 = ngx.encode_base64
 local concat = table.concat
 local char = string.char
 local str_find = string.find
+local str_sub = string.sub
 local rand = math.random
 local rshift = bit.rshift
 local band = bit.band
@@ -159,22 +160,28 @@ function _M.connect(self, uri, opts)
     end
 
     if proxy_url then
-        -- https://github.com/ledgetech/lua-resty-http/blob/master/lib/resty/http.lua
-        local m, err = re_match(
-            proxy_url,
-            [[^(?:(http[s]?):)?//((?:[^\[\]:/\?]+)|(?:\[.+\]))(?::(\d+))?([^\?]*)\??(.*)]],
-            "jo"
-        )
-        if err then
-            return nil, "error parsing proxy_url: " .. err
+        if str_sub(proxy_url, 1, 6) == "unix:/" then
+            connect_host = proxy_url
+            connect_port = nil
+
+        else
+            -- https://github.com/ledgetech/lua-resty-http/blob/master/lib/resty/http.lua
+            local m, err = re_match(
+                proxy_url,
+                [[^(?:(http[s]?):)?//((?:[^\[\]:/\?]+)|(?:\[.+\]))(?::(\d+))?([^\?]*)\??(.*)]],
+                "jo"
+            )
+            if err then
+                return nil, "error parsing proxy_url: " .. err
+
+            elseif m[1] ~= "http" and m[1] ~= "https" then
+                return nil, "only proxy with scheme \"http\" or \"https\" is supported"
+            end
 
-        elseif m[1] ~= "http" then
-            return nil, "only HTTP proxy is supported"
+            connect_host = m[2]
+            connect_port = m[3] or 443
         end
 
-        connect_host = m[2]
-        connect_port = m[3] or 80 -- hardcode for now as we only support HTTP proxy
-
         if not connect_host then
             return nil, "invalid proxy url"
         end

kong/templates/kong_defaults.lua

@@ -15,6 +15,7 @@ port_maps = NONE
 host_ports = NONE
 anonymous_reports = on
 proxy_server = NONE
+proxy_server_ssl_verify = on
 
 proxy_listen = 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384
 stream_listen = off

kong/templates/nginx.lua

@@ -25,9 +25,33 @@ http {
 }
 > end
 
-> if #stream_listeners > 0 then
+> if #stream_listeners > 0 or cluster_ssl_tunnel then
 stream {
+> if #stream_listeners > 0 then
     include 'nginx-kong-stream.conf';
+> end
+
+> if cluster_ssl_tunnel then
+    server {
+        listen unix:${{PREFIX}}/cluster_proxy_ssl_terminator.sock;
+
+        proxy_pass ${{cluster_ssl_tunnel}};
+        proxy_ssl on;
+        # as we are essentially talking in HTTPS, passing SNI should default turned on
+        proxy_ssl_server_name on;
+> if proxy_server_ssl_verify then
+        proxy_ssl_verify on;
+> if lua_ssl_trusted_certificate_combined then
+        proxy_ssl_trusted_certificate '${{LUA_SSL_TRUSTED_CERTIFICATE_COMBINED}}';
+> end
+        proxy_ssl_verify_depth 5; # 5 should be sufficient
+> else
+        proxy_ssl_verify off;
+> end
+        proxy_socket_keepalive on;
+    }
+> end -- cluster_ssl_tunnel
+
 }
 > end
 ]]

spec/01-unit/03-conf_loader_spec.lua

@@ -987,7 +987,7 @@ describe("Configuration loader", function()
           local conf, _, errors = conf_loader(nil, {
             proxy_server = "cool://localhost:2333",
           })
-          assert.contains("proxy_server only supports \"http\", got cool", errors)
+          assert.contains("proxy_server only supports \"http\" and \"https\", got cool", errors)
           assert.is_nil(conf)
 
           local conf, _, errors = conf_loader(nil, {

spec/02-integration/09-hybrid_mode/10-forward-proxy_spec.lua

@@ -8,6 +8,12 @@ local fixtures = {
     forward_proxy = [[
     server {
       listen 16797;
+      listen 16799 ssl;
+      listen [::]:16799 ssl;
+
+      ssl_certificate ../spec/fixtures/kong_spec.crt;
+      ssl_certificate_key ../spec/fixtures/kong_spec.key;
+
       error_log logs/proxy.log debug;
 
       content_by_lua_block {
@@ -17,29 +23,57 @@ local fixtures = {
 
     server {
       listen 16796;
+      listen 16798 ssl;
+      listen [::]:16798 ssl;
+
+      ssl_certificate ../spec/fixtures/kong_spec.crt;
+      ssl_certificate_key ../spec/fixtures/kong_spec.key;
+
       error_log logs/proxy_auth.log debug;
 
+
       content_by_lua_block {
         require("spec.fixtures.forward-proxy-server").connect({
           basic_auth = ngx.encode_base64("test:konghq"),
         })
       }
     }
-
     ]],
   },
 }
 
 
-local auth_confgs = {
-  ["auth off"] = "http://127.0.0.1:16797",
-  ["auth on"] = "http://test:[email protected]:16796",
+local proxy_configs = {
+  ["https off auth off"] = {
+    proxy_server = "http://127.0.0.1:16797",
+    proxy_server_ssl_verify = "off",
+  },
+  ["https off auth on"] = {
+    proxy_server = "http://test:[email protected]:16796",
+    proxy_server_ssl_verify = "off",
+  },
+  ["https on auth off"] = {
+    proxy_server = "https://127.0.0.1:16799",
+    proxy_server_ssl_verify = "off",
+  },
+  ["https on auth on"] = {
+    proxy_server = "https://test:[email protected]:16798",
+    proxy_server_ssl_verify = "off",
+  },
+  ["https on auth off verify on"] = {
+    proxy_server = "https://localhost:16799", -- use `localhost` to match CN of cert
+    proxy_server_ssl_verify = "on",
+    lua_ssl_trusted_certificate = "spec/fixtures/kong_spec.crt",
+  },
 }
 
+-- Note: this test suite will become flakky if KONG_TEST_DONT_CLEAN
+-- if existing lmdb data is set, the service/route exists and
+-- test run too fast before the proxy connection is established
 
 for _, strategy in helpers.each_strategy() do
-  for auth_desc, proxy_url in pairs(auth_confgs) do
-    describe("CP/DP sync through proxy (" .. auth_desc .. ") works with #" .. strategy .. " backend", function()
+  for proxy_desc, proxy_opts in pairs(proxy_configs) do
+    describe("CP/DP sync through proxy (" .. proxy_desc .. ") works with #" .. strategy .. " backend", function()
       lazy_setup(function()
         helpers.get_db_utils(strategy) -- runs migrations
 
@@ -67,7 +101,9 @@ for _, strategy in helpers.each_strategy() do
           nginx_conf = "spec/fixtures/custom_nginx.template",
 
           cluster_use_proxy = "on",
-          proxy_server = proxy_url,
+          proxy_server = proxy_opts.proxy_server,
+          proxy_server_ssl_verify = proxy_opts.proxy_server_ssl_verify,
+          lua_ssl_trusted_certificate = proxy_opts.lua_ssl_trusted_certificate,
 
           -- this is unused, but required for the the template to include a stream {} block
           stream_listen = "0.0.0.0:5555",
@@ -114,18 +150,20 @@ for _, strategy in helpers.each_strategy() do
             end
           end, 10)
 
+          local auth_on = string.match(proxy_desc, "auth on")
+
           -- ensure this goes through proxy
           local path = pl_path.join("servroot2", "logs",
-                      (auth_desc == "auth on") and "proxy_auth.log" or "proxy.log")
+                                    auth_on and "proxy_auth.log" or "proxy.log")
           local contents = pl_file.read(path)
           assert.matches("CONNECT 127.0.0.1:9005", contents)
 
-          if auth_desc == "auth on" then
+          if auth_on then
             assert.matches("accepted basic proxy%-authorization", contents)
           end
         end)
       end)
     end)
 
-  end -- auth configs 
+  end -- proxy configs
 end

spec/fixtures/custom_nginx.template

@@ -724,7 +724,7 @@ http {
 }
 > end
 
-> if #stream_listeners > 0 then
+> if #stream_listeners > 0 or cluster_ssl_tunnel then
 stream {
     log_format basic '$remote_addr [$time_local] '
                      '$protocol $status $bytes_sent $bytes_received '
@@ -968,5 +968,26 @@ server {
     }
 > end -- not legacy_worker_events
 
+> if cluster_ssl_tunnel then
+    server {
+        listen unix:${{PREFIX}}/cluster_proxy_ssl_terminator.sock;
+
+        proxy_pass ${{cluster_ssl_tunnel}};
+        proxy_ssl on;
+        # as we are essentially talking in HTTPS, passing SNI should default turned on
+        proxy_ssl_server_name on;
+> if proxy_server_ssl_verify then
+        proxy_ssl_verify on;
+> if lua_ssl_trusted_certificate_combined then
+        proxy_ssl_trusted_certificate '${{LUA_SSL_TRUSTED_CERTIFICATE_COMBINED}}';
+> end
+        proxy_ssl_verify_depth 5; # 5 should be sufficient
+> else
+        proxy_ssl_verify off;
+> end
+        proxy_socket_keepalive on;
+    }
+> end -- cluster_ssl_tunnel
+
 }
 > end