HCM: add support for IP detection extensions

api/envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.proto

@@ -35,7 +35,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // HTTP connection manager :ref:`configuration overview <config_http_conn_man>`.
 // [#extension: envoy.filters.network.http_connection_manager]
 
-// [#next-free-field: 43]
+// [#next-free-field: 44]
 message HttpConnectionManager {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager";
@@ -583,6 +583,11 @@ message HttpConnectionManager {
   // *not* the deprecated but similarly named :ref:`stream_error_on_invalid_http_messaging
   // <envoy_v3_api_field_config.core.v3.Http2ProtocolOptions.stream_error_on_invalid_http_messaging>`
   google.protobuf.BoolValue stream_error_on_invalid_http_message = 40;
+
+  // The configuration of the IP detection extension.
+  //
+  // If not set, Envoy uses the default remote IP detection.
+  IPDetectionExtension ip_detection_extension = 43;
 }
 
 // The configuration to customize local reply returned by Envoy.
@@ -851,3 +856,8 @@ message RequestIDExtension {
   // Request ID extension specific configuration.
   google.protobuf.Any typed_config = 1;
 }
+
+message IPDetectionExtension {
+  // IP detection extension specific configuration.
+  google.protobuf.Any typed_config = 1;
+}

docs/root/version_history/current.rst

@@ -54,6 +54,7 @@ New Features
 * access log: added the :ref:`formatters <envoy_v3_api_field_config.core.v3.SubstitutionFormatString.formatters>` extension point for custom formatters (command operators).
 * access log: support command operator: %REQUEST_HEADERS_BYTES%, %RESPONSE_HEADERS_BYTES% and %RESPONSE_TRAILERS_BYTES%.
 * dispatcher: supports a stack of `Envoy::ScopeTrackedObject` instead of a single tracked object. This will allow Envoy to dump more debug information on crash.
+* http: added support for IP detection extensions.
 * http: added support for :ref:`:ref:`preconnecting <envoy_v3_api_msg_config.cluster.v3.Cluster.PreconnectPolicy>`. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic, especially if using HTTP/1.1.
 * http: change frame flood and abuse checks to the upstream HTTP/2 codec to ON by default. It can be disabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to false.
 * overload: add support for scaling :ref:`transport connection timeouts<envoy_v3_api_enum_value_config.overload.v3.ScaleTimersOverloadActionConfig.TimerType.TRANSPORT_SOCKET_CONNECT>`. This can be used to reduce the TLS handshake timeout in response to overload.

include/envoy/http/BUILD

@@ -140,3 +140,12 @@ envoy_cc_library(
         ":header_map_interface",
     ],
 )
+
+envoy_cc_library(
+    name = "ip_detection_extension_interface",
+    hdrs = ["ip_detection_extension.h"],
+    deps = [
+        ":header_map_interface",
+        "//include/envoy/config:typed_config_interface",
+    ],
+)

include/envoy/http/ip_detection_extension.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "envoy/common/pure.h"
+#include "envoy/http/header_map.h"
+
+namespace Envoy {
+namespace Http {
+
+/**
+ * Interface class for IP detection extensions.
+ */
+class IPDetectionExtension {
+public:
+  virtual ~IPDetectionExtension() = default;
+
+  /**
+   * Detect the final remote address if any.
+   *
+   * @param request_headers supplies the incoming request headers.
+   */
+  virtual Network::Address::InstanceConstSharedPtr
+  detect(Http::RequestHeaderMap& request_headers) PURE;
+};
+
+using IPDetectionExtensionSharedPtr = std::shared_ptr<IPDetectionExtension>;
+
+/*
+ * A factory for creating IP detection extensions.
+ */
+class IPDetectionExtensionFactory : public Envoy::Config::TypedFactory {
+public:
+  ~IPDetectionExtensionFactory() override = default;
+
+  /**
+   * Creates a particular Extension implementation.
+   *
+   * @param config supplies the configuration for the IP detection extension.
+   * @return IPDetectionExtensionSharedPtr the extension instance.
+   */
+  virtual IPDetectionExtensionSharedPtr createExtension(const Protobuf::Message& config) const PURE;
+};
+
+using IPDetectionExtensionFactoryPtr = std::unique_ptr<IPDetectionExtensionFactory>;
+
+} // namespace Http
+} // namespace Envoy

source/common/http/BUILD

@@ -155,6 +155,7 @@ envoy_cc_library(
         ":date_provider_lib",
         "//include/envoy/config:config_provider_interface",
         "//include/envoy/http:filter_interface",
+        "//include/envoy/http:ip_detection_extension_interface",
         "//include/envoy/http:request_id_extension_interface",
         "//include/envoy/router:rds_interface",
         "//source/common/local_reply:local_reply_lib",

source/common/http/conn_manager_config.h

@@ -3,6 +3,7 @@
 #include "envoy/config/config_provider.h"
 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h"
 #include "envoy/http/filter.h"
+#include "envoy/http/ip_detection_extension.h"
 #include "envoy/http/request_id_extension.h"
 #include "envoy/router/rds.h"
 #include "envoy/stats/scope.h"
@@ -466,6 +467,11 @@ class ConnectionManagerConfig {
    * @return LocalReply configuration which supplies mapping for local reply generated by Envoy.
    */
   virtual const LocalReply::LocalReply& localReply() const PURE;
+
+  /**
+   * @return IPDetectionExtensionSharedPtr The IP detection extension if available.
+   */
+  virtual IPDetectionExtensionSharedPtr ipDetectionExtension() PURE;
 };
 } // namespace Http
 } // namespace Envoy

source/common/http/conn_manager_utility.cc

@@ -95,7 +95,7 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest
   // peer. Cases where we don't "use remote address" include trusted double proxy where we expect
   // our peer to have already properly set XFF, etc.
   Network::Address::InstanceConstSharedPtr final_remote_address;
-  bool single_xff_address;
+  bool single_xff_address = false;
   const uint32_t xff_num_trusted_hops = config.xffNumTrustedHops();
 
   if (config.useRemoteAddress()) {
@@ -127,12 +127,23 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest
           connection.ssl() ? Headers::get().SchemeValues.Https : Headers::get().SchemeValues.Http);
     }
   } else {
-    // If we are not using remote address, attempt to pull a valid IPv4 or IPv6 address out of XFF.
+    // If we are not using remote address, attempt to pull a valid IPv4 or IPv6 address out of XFF
+    // or through an extension. An extension might be needed when XFF doesn't work (e.g. an
+    // irregular network).
+    //
     // If we find one, it will be used as the downstream address for logging. It may or may not be
     // used for determining internal/external status (see below).
-    auto ret = Utility::getLastAddressFromXFF(request_headers, xff_num_trusted_hops);
-    final_remote_address = ret.address_;
-    single_xff_address = ret.single_address_;
+    auto ip_detection_extension = config.ipDetectionExtension();
+    if (ip_detection_extension) {
+      final_remote_address = ip_detection_extension->detect(request_headers);
+    }
+
+    // If there's no extension or it failed to detect, give XFF a try.
+    if (!final_remote_address) {
+      auto ret = Utility::getLastAddressFromXFF(request_headers, xff_num_trusted_hops);
+      final_remote_address = ret.address_;
+      single_xff_address = ret.single_address_;
+    }
   }
 
   // If the x-forwarded-proto header is not set, set it here, since Envoy uses it for determining
@@ -170,30 +181,7 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest
     request_headers.setReferenceEnvoyInternalRequest(
         Headers::get().EnvoyInternalRequestValues.True);
   } else {
-    if (edge_request) {
-      request_headers.removeEnvoyDecoratorOperation();
-      request_headers.removeEnvoyDownstreamServiceCluster();
-      request_headers.removeEnvoyDownstreamServiceNode();
-    }
-
-    request_headers.removeEnvoyRetriableStatusCodes();
-    request_headers.removeEnvoyRetriableHeaderNames();
-    request_headers.removeEnvoyRetryOn();
-    request_headers.removeEnvoyRetryGrpcOn();
-    request_headers.removeEnvoyMaxRetries();
-    request_headers.removeEnvoyUpstreamAltStatName();
-    request_headers.removeEnvoyUpstreamRequestTimeoutMs();
-    request_headers.removeEnvoyUpstreamRequestPerTryTimeoutMs();
-    request_headers.removeEnvoyUpstreamRequestTimeoutAltResponse();
-    request_headers.removeEnvoyExpectedRequestTimeoutMs();
-    request_headers.removeEnvoyForceTrace();
-    request_headers.removeEnvoyIpTags();
-    request_headers.removeEnvoyOriginalUrl();
-    request_headers.removeEnvoyHedgeOnPerTryTimeout();
-
-    for (const LowerCaseString& header : route_config.internalOnlyHeaders()) {
-      request_headers.remove(header);
-    }
+    cleanInternalHeaders(request_headers, edge_request, route_config.internalOnlyHeaders());
   }
 
   if (config.userAgent()) {
@@ -236,6 +224,35 @@ Network::Address::InstanceConstSharedPtr ConnectionManagerUtility::mutateRequest
   return final_remote_address;
 }
 
+void ConnectionManagerUtility::cleanInternalHeaders(
+    RequestHeaderMap& request_headers, bool edge_request,
+    const std::list<Http::LowerCaseString>& internal_only_headers) {
+  if (edge_request) {
+    request_headers.removeEnvoyDecoratorOperation();
+    request_headers.removeEnvoyDownstreamServiceCluster();
+    request_headers.removeEnvoyDownstreamServiceNode();
+  }
+
+  request_headers.removeEnvoyRetriableStatusCodes();
+  request_headers.removeEnvoyRetriableHeaderNames();
+  request_headers.removeEnvoyRetryOn();
+  request_headers.removeEnvoyRetryGrpcOn();
+  request_headers.removeEnvoyMaxRetries();
+  request_headers.removeEnvoyUpstreamAltStatName();
+  request_headers.removeEnvoyUpstreamRequestTimeoutMs();
+  request_headers.removeEnvoyUpstreamRequestPerTryTimeoutMs();
+  request_headers.removeEnvoyUpstreamRequestTimeoutAltResponse();
+  request_headers.removeEnvoyExpectedRequestTimeoutMs();
+  request_headers.removeEnvoyForceTrace();
+  request_headers.removeEnvoyIpTags();
+  request_headers.removeEnvoyOriginalUrl();
+  request_headers.removeEnvoyHedgeOnPerTryTimeout();
+
+  for (const LowerCaseString& header : internal_only_headers) {
+    request_headers.remove(header);
+  }
+}
+
 void ConnectionManagerUtility::mutateTracingRequestHeader(RequestHeaderMap& request_headers,
                                                           Runtime::Loader& runtime,
                                                           ConnectionManagerConfig& config,

source/common/http/conn_manager_utility.h

@@ -87,6 +87,8 @@ class ConnectionManagerUtility {
   static void mutateXfccRequestHeader(RequestHeaderMap& request_headers,
                                       Network::Connection& connection,
                                       ConnectionManagerConfig& config);
+  static void cleanInternalHeaders(RequestHeaderMap& request_headers, bool edge_request,
+                                   const std::list<Http::LowerCaseString>& internal_only_headers);
 };
 
 } // namespace Http

source/extensions/filters/network/http_connection_manager/BUILD

@@ -24,6 +24,7 @@ envoy_cc_extension(
         "//include/envoy/filesystem:filesystem_interface",
         "//include/envoy/http:codec_interface",
         "//include/envoy/http:filter_interface",
+        "//include/envoy/http:ip_detection_extension_interface",
         "//include/envoy/http:request_id_extension_interface",
         "//include/envoy/registry",
         "//include/envoy/router:route_config_provider_manager_interface",

source/extensions/filters/network/http_connection_manager/config.cc

@@ -287,6 +287,18 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig(
         Http::RequestIDExtensionFactory::defaultInstance(context_.api().randomGenerator());
   }
 
+  // Check if we are provided with an IP detection extension.
+  if (config.ip_detection_extension().has_typed_config()) {
+    auto& typed_config = config.ip_detection_extension().typed_config();
+    const std::string type{TypeUtil::typeUrlToDescriptorFullName(typed_config.type_url())};
+    auto* ip_detection_extension_factory =
+        Registry::FactoryRegistry<Http::IPDetectionExtensionFactory>::getFactoryByType(type);
+    if (!ip_detection_extension_factory) {
+      throw EnvoyException("IP detection extension not found");
+    }
+    ip_detection_extension_ = ip_detection_extension_factory->createExtension(typed_config);
+  }
+
   // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will
   // be managed by the scoped routing logic instead.
   switch (config.route_specifier_case()) {

source/extensions/filters/network/http_connection_manager/config.h

@@ -13,6 +13,7 @@
 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.validate.h"
 #include "envoy/filter/http/filter_config_provider.h"
 #include "envoy/http/filter.h"
+#include "envoy/http/ip_detection_extension.h"
 #include "envoy/http/request_id_extension.h"
 #include "envoy/router/route_config_provider_manager.h"
 #include "envoy/tracing/http_tracer_manager.h"
@@ -177,6 +178,9 @@ class HttpConnectionManagerConfig : Logger::Loggable<Logger::Id::config>,
   }
   std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; }
   const LocalReply::LocalReply& localReply() const override { return *local_reply_; }
+  Http::IPDetectionExtensionSharedPtr ipDetectionExtension() override {
+    return ip_detection_extension_;
+  }
 
 private:
   enum class CodecType { HTTP1, HTTP2, HTTP3, AUTO };
@@ -255,6 +259,7 @@ class HttpConnectionManagerConfig : Logger::Loggable<Logger::Id::config>,
   const envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction
       headers_with_underscores_action_;
   const LocalReply::LocalReplyPtr local_reply_;
+  Http::IPDetectionExtensionSharedPtr ip_detection_extension_{nullptr};
 
   // Default idle timeout is 5 minutes if nothing is specified in the HCM config.
   static const uint64_t StreamIdleTimeoutMs = 5 * 60 * 1000;

source/server/admin/admin.h

@@ -180,6 +180,7 @@ class AdminImpl : public Admin,
     return envoy::config::core::v3::HttpProtocolOptions::ALLOW;
   }
   const LocalReply::LocalReply& localReply() const override { return *local_reply_; }
+  Http::IPDetectionExtensionSharedPtr ipDetectionExtension() override { return nullptr; };
   Http::Code request(absl::string_view path_and_query, absl::string_view method,
                      Http::ResponseHeaderMap& response_headers, std::string& body) override;
   void closeSocket();