feat(clustering) support http forward proxy

CHANGELOG.md

@@ -159,6 +159,10 @@
   worker.
   [#9616](https://github.com/Kong/kong/pull/9616)
 
+- Add HTTP CONNECT forward proxy support for Hybrid Mode connections. New configuration
+  options `cluster_use_proxy` and `proxy_server` are added.
+  [#9758](https://github.com/Kong/kong/pull/9758)
+
 #### CLI
 
 - Fix slow CLI performance due to pending timer jobs

kong-3.1.0-0.rockspec

@@ -92,6 +92,7 @@ build = {
     ["kong.resty.dns.client"] = "kong/resty/dns/client.lua",
     ["kong.resty.dns.utils"] = "kong/resty/dns/utils.lua",
     ["kong.resty.ctx"] = "kong/resty/ctx.lua",
+    ["kong.resty.websocket.client"] = "kong/resty/websocket/client.lua",
 
     ["kong.cmd"] = "kong/cmd/init.lua",
     ["kong.cmd.roar"] = "kong/cmd/roar.lua",

kong.conf.default

@@ -221,6 +221,10 @@
 #anonymous_reports = on          # Send anonymous usage data such as error
                                  # stack traces to help improve Kong.
 
+
+#proxy_server =                  # Proxy server defined as a URL. Kong will only use
+                                 # option if any component is explictly configured
+                                 # to use proxy.
 #------------------------------------------------------------------------------
 # HYBRID MODE
 #------------------------------------------------------------------------------
@@ -390,6 +394,10 @@
                          #   found inside DP provided certificate
                          #   or communication with the OCSP responder
                          #   failed, then DP is still allowed through.
+#cluster_use_proxy = off
+                         # Whether to turn on HTTP CONNECT proxy support for
+                         # hybrid mode connections. `proxy_server` will be used
+                         # for Hybrid mode connections if this option is turned on.
 #------------------------------------------------------------------------------
 # NGINX
 #------------------------------------------------------------------------------

kong/clustering/utils.lua

@@ -5,10 +5,11 @@ local openssl_x509 = require("resty.openssl.x509")
 local ssl = require("ngx.ssl")
 local ocsp = require("ngx.ocsp")
 local http = require("resty.http")
-local ws_client = require("resty.websocket.client")
+local ws_client = require("kong.resty.websocket.client")
 local ws_server = require("resty.websocket.server")
 local utils = require("kong.tools.utils")
 local meta = require("kong.meta")
+local parse_url = require "socket.url".parse
 
 local type = type
 local tonumber = tonumber
@@ -17,12 +18,15 @@ local table_insert = table.insert
 local table_concat = table.concat
 local gsub = string.gsub
 local process_type = require("ngx.process").type
+local encode_base64 = ngx.encode_base64
+local fmt = string.format
 
 local kong = kong
 
 local ngx = ngx
 local ngx_var = ngx.var
 local ngx_log = ngx.log
+local ngx_DEBUG = ngx.DEBUG
 local ngx_INFO = ngx.INFO
 local ngx_NOTICE = ngx.NOTICE
 local ngx_WARN = ngx.WARN
@@ -324,6 +328,25 @@ function _M.check_configuration_compatibility(obj, dp_plugin_map)
   return true, nil, CLUSTERING_SYNC_STATUS.NORMAL
 end
 
+local function parse_proxy_url(conf)
+  local ret = {}
+  local proxy_server = conf.proxy_server
+  if proxy_server then
+    -- assume proxy_server is validated in conf_loader
+    local parsed = parse_url(proxy_server)
+    ret.proxy_url = fmt("%s://%s:%s", parsed.scheme, parsed.host, parsed.port or 443)
+    ret.scheme = parsed.scheme
+    ret.host = parsed.host
+    ret.port = parsed.port
+
+    if parsed.user and parsed.password then
+      ret.proxy_authorization = "Basic " .. encode_base64(parsed.user  .. ":" .. parsed.password)
+    end
+  end
+
+  return ret
+end
+
 
 --- Return the highest supported Hybrid mode protocol version.
 function _M.check_protocol_support(conf, cert, cert_key)
@@ -347,6 +370,18 @@ function _M.check_protocol_support(conf, cert, cert_key)
   end
 
   local c = http.new()
+
+  if conf.cluster_use_proxy then
+    local proxy_opts = parse_proxy_url(conf)
+    c:set_proxy_options({
+      https_proxy = proxy_opts.proxy_url,
+      https_proxy_authorization = proxy_opts.proxy_authorization,
+    })
+
+    ngx_log(ngx_DEBUG, _log_prefix,
+            "using proxy ", proxy_opts.proxy_url, " to check protocol support ")
+  end
+
   local res, err = c:request_uri(
     "https://" .. conf.cluster_control_plane .. "/v1/wrpc", params)
   if not res then
@@ -383,6 +418,17 @@ function _M.connect_cp(endpoint, conf, cert, cert_key, protocols)
     protocols = protocols,
   }
 
+  if conf.cluster_use_proxy then
+    local proxy_opts = parse_proxy_url(conf)
+    opts.proxy_opts = {
+      wss_proxy = proxy_opts.proxy_url,
+      wss_proxy_authorization = proxy_opts.proxy_authorization,
+    }
+
+    ngx_log(ngx_DEBUG, _log_prefix,
+            "using proxy ", proxy_opts.proxy_url, " to connect control plane")
+  end
+
   if conf.cluster_mtls == "shared" then
     opts.server_name = "kong_clustering"
 

kong/conf_loader/init.lua

@@ -7,6 +7,7 @@ local openssl_pkey = require "resty.openssl.pkey"
 local openssl_x509 = require "resty.openssl.x509"
 local pl_stringio = require "pl.stringio"
 local pl_stringx = require "pl.stringx"
+local socket_url = require "socket.url"
 local constants = require "kong.constants"
 local listeners = require "kong.conf_loader.listeners"
 local pl_pretty = require "pl.pretty"
@@ -524,6 +525,7 @@ local CONF_INFERENCES = {
   cluster_data_plane_purge_delay = { typ = "number" },
   cluster_ocsp = { enum = { "on", "off", "optional" } },
   cluster_max_payload = { typ = "number" },
+  cluster_use_proxy = { typ = "boolean" },
 
   kic = { typ = "boolean" },
   pluginserver_names = { typ = "array" },
@@ -539,6 +541,8 @@ local CONF_INFERENCES = {
 
   opentelemetry_tracing = { typ = "array" },
   opentelemetry_tracing_sampling_rate = { typ = "number" },
+
+  proxy_server = { typ = "string" },
 }
 
 
@@ -549,6 +553,7 @@ local CONF_SENSITIVE = {
   pg_password = true,
   pg_ro_password = true,
   cassandra_password = true,
+  proxy_server = true, -- hide proxy server URL as it may contain credentials
 }
 
 
@@ -982,6 +987,25 @@ local function check_and_infer(conf, opts)
     errors[#errors + 1] = "worker_state_update_frequency must be greater than 0"
   end
 
+  if conf.proxy_server then
+    local parsed, err = socket_url.parse(conf.proxy_server)
+    if err then
+      errors[#errors + 1] = "proxy_server is invalid: " .. err
+
+    elseif not parsed.scheme then
+      errors[#errors + 1] = "proxy_server missing scheme"
+
+    elseif parsed.scheme ~= "http" then
+      errors[#errors + 1] = "proxy_server only supports \"http\", got " .. parsed.scheme
+
+    elseif not parsed.host then
+      errors[#errors + 1] = "proxy_server missing host"
+
+    elseif parsed.fragment or parsed.query or parsed.params then
+      errors[#errors + 1] = "fragments, query strings or parameters are meaningless in proxy configuration"
+    end
+  end
+
   if conf.role == "control_plane" then
     if #conf.admin_listen < 1 or strip(conf.admin_listen[1]) == "off" then
       errors[#errors + 1] = "admin_listen must be specified when role = \"control_plane\""
@@ -999,6 +1023,10 @@ local function check_and_infer(conf, opts)
       errors[#errors + 1] = "in-memory storage can not be used when role = \"control_plane\""
     end
 
+    if conf.cluster_use_proxy then
+      errors[#errors + 1] = "cluster_use_proxy can not be used when role = \"control_plane\""
+    end
+
   elseif conf.role == "data_plane" then
     if #conf.proxy_listen < 1 or strip(conf.proxy_listen[1]) == "off" then
       errors[#errors + 1] = "proxy_listen must be specified when role = \"data_plane\""
@@ -1019,6 +1047,10 @@ local function check_and_infer(conf, opts)
     elseif conf.cluster_mtls == "pki" then
       insert(conf.lua_ssl_trusted_certificate, conf.cluster_ca_cert)
     end
+
+    if conf.cluster_use_proxy and not conf.proxy_server then
+      errors[#errors + 1] = "cluster_use_proxy is turned on but no proxy_server is configured"
+    end
   end
 
   if conf.cluster_data_plane_purge_delay < 60 then