http: add CONNECT-UDP support (envoyproxy#27714)

Commit Message: This commit adds CONNECT-UDP (RFC 9298) support. UdpConnPool is added to create a UDP socket for a new CONNECT-UDP request, and UDPUpstream is added to maintain the socket and other relevant data associated with UDP upstreams. We added an integration test for the terminating CONNECT-UDP proxy, but not the forwarding proxy in this commit. We are going to add test cases to cover the forwarding proxy scenario in a subsequent commit. Additional Description: Risk Level: Medium, the feature can only be enabled by the new configuration added in this commit. Testing: Integration test Runtime guard: envoy.reloadable_features.enable_connect_udp_support Release Notes: added support for CONNECT-UDP (RFC 9298). Can be disabled by setting runtime feature envoy.reloadable_features.enable_connect_udp_support to false. Signed-off-by: Jeongseok Son <[email protected]> Co-authored-by: asingh-g <[email protected]>
amolbms · Jun 22, 2023 · b4f3755 · b4f3755
1 parent 9c6e750
commit b4f3755
Show file tree

Hide file tree

Showing 62 changed files with 1,371 additions and 81 deletions.
diff --git a/api/BUILD b/api/BUILD
@@ -309,6 +309,7 @@ proto_library(
         "//envoy/extensions/upstreams/http/generic/v3:pkg",
         "//envoy/extensions/upstreams/http/http/v3:pkg",
         "//envoy/extensions/upstreams/http/tcp/v3:pkg",
+        "//envoy/extensions/upstreams/http/udp/v3:pkg",
         "//envoy/extensions/upstreams/http/v3:pkg",
         "//envoy/extensions/upstreams/tcp/generic/v3:pkg",
         "//envoy/extensions/upstreams/tcp/v3:pkg",

diff --git a/api/envoy/extensions/upstreams/http/udp/v3/BUILD b/api/envoy/extensions/upstreams/http/udp/v3/BUILD
@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+)
diff --git a/api/envoy/extensions/upstreams/http/udp/v3/udp_connection_pool.proto b/api/envoy/extensions/upstreams/http/udp/v3/udp_connection_pool.proto
@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+package envoy.extensions.upstreams.http.udp.v3;
+
+import "udpa/annotations/status.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.upstreams.http.udp.v3";
+option java_outer_classname = "UdpConnectionPoolProtoOuterClass";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/udp/v3;udpv3";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Udp Connection Pool]
+
+// A connection pool which forwards downstream HTTP as UDP to upstream,
+// [#extension: envoy.upstreams.http.udp]
+message UdpConnectionPoolProto {
+}
diff --git a/api/versioning/BUILD b/api/versioning/BUILD
@@ -248,6 +248,7 @@ proto_library(
         "//envoy/extensions/upstreams/http/generic/v3:pkg",
         "//envoy/extensions/upstreams/http/http/v3:pkg",
         "//envoy/extensions/upstreams/http/tcp/v3:pkg",
+        "//envoy/extensions/upstreams/http/udp/v3:pkg",
         "//envoy/extensions/upstreams/http/v3:pkg",
         "//envoy/extensions/upstreams/tcp/generic/v3:pkg",
         "//envoy/extensions/upstreams/tcp/v3:pkg",

diff --git a/bazel/external/quiche.BUILD b/bazel/external/quiche.BUILD
@@ -3177,6 +3177,21 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "quiche_common_connect_udp_datagram_payload_lib",
+    srcs = ["quiche/common/masque/connect_udp_datagram_payload.cc"],
+    hdrs = ["quiche/common/masque/connect_udp_datagram_payload.h"],
+    copts = quiche_copts,
+    repository = "@envoy",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":quic_core_data_lib",
+        ":quiche_common_platform_bug_tracker",
+        ":quiche_common_platform_logging",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 envoy_cc_library(
     name = "quiche_common_quiche_stream_lib",
     srcs = [],

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
@@ -240,6 +240,10 @@ new_features:
     added Runtime feature ``envoy.reloadable_features.max_request_headers_size_kb`` to override the default value of
     :ref:`max request headers size
     <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.max_request_headers_kb>`.
+- area: http
+  change: |
+    added support for CONNECT-UDP (RFC 9298). Can be disabled by setting runtime feature
+    ``envoy.reloadable_features.enable_connect_udp_support`` to false.
 - area: listeners
   change: |
     added :ref:`max_connections_to_accept_per_socket_event

diff --git a/envoy/network/connection.h b/envoy/network/connection.h
@@ -149,7 +149,7 @@ class Connection : public Event::DeferredDeletable,
   /**
    * @return Event::Dispatcher& the dispatcher backing this connection.
    */
-  virtual Event::Dispatcher& dispatcher() PURE;
+  virtual Event::Dispatcher& dispatcher() const PURE;
 
   /**
    * @return uint64_t the unique local ID of this connection.

diff --git a/envoy/router/router.h b/envoy/router/router.h
@@ -1523,14 +1523,20 @@ using GenericConnPoolPtr = std::unique_ptr<GenericConnPool>;
  */
 class GenericConnPoolFactory : public Envoy::Config::TypedFactory {
 public:
+  /*
+   * Protocol used by the upstream sockets.
+   */
+  enum class UpstreamProtocol { HTTP, TCP, UDP };
+
   ~GenericConnPoolFactory() override = default;
 
   /*
    * @param options for creating the transport socket
    * @return may be null
    */
   virtual GenericConnPoolPtr
-  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster, bool is_connect,
+  createGenericConnPool(Upstream::ThreadLocalCluster& thread_local_cluster,
+                        GenericConnPoolFactory::UpstreamProtocol upstream_protocol,
                         const RouteEntry& route_entry,
                         absl::optional<Http::Protocol> downstream_protocol,
                         Upstream::LoadBalancerContext* ctx) const PURE;

diff --git a/envoy/stream_info/stream_info.h b/envoy/stream_info/stream_info.h
@@ -135,6 +135,8 @@ struct ResponseCodeDetailValues {
   const std::string InvalidEnvoyRequestHeaders = "request_headers_failed_strict_check";
   // The request was rejected due to a missing Path or :path header field.
   const std::string MissingPath = "missing_path_rejected";
+  // The request was rejected due to an invalid Path or :path header field.
+  const std::string InvalidPath = "invalid_path";
   // The request was rejected due to using an absolute path on a route not supporting them.
   const std::string AbsolutePath = "absolute_path_rejected";
   // The request was rejected because path normalization was configured on and failed, probably due

diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc
@@ -1177,6 +1177,15 @@ void ConnectionManagerImpl::ActiveStream::decodeHeaders(RequestHeaderMapSharedPt
     return;
   }
 
+  // Rewrites the host of CONNECT-UDP requests.
+  if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") &&
+      HeaderUtility::isConnectUdp(*request_headers_) &&
+      !HeaderUtility::rewriteAuthorityForConnectUdp(*request_headers_)) {
+    sendLocalReply(Code::NotFound, "The path is incorrect for CONNECT-UDP", nullptr, absl::nullopt,
+                   StreamInfo::ResponseCodeDetails::get().InvalidPath);
+    return;
+  }
+
   // Currently we only support relative paths at the application layer.
   if (!request_headers_->getPathValue().empty() && request_headers_->getPathValue()[0] != '/') {
     connection_manager_.stats_.named_.downstream_rq_non_relative_path_.inc();

diff --git a/source/common/http/header_utility.cc b/source/common/http/header_utility.cc
@@ -243,13 +243,62 @@ bool HeaderUtility::isConnect(const RequestHeaderMap& headers) {
   return headers.Method() && headers.Method()->value() == Http::Headers::get().MethodValues.Connect;
 }
 
+bool HeaderUtility::isConnectUdp(const RequestHeaderMap& headers) {
+  return headers.Upgrade() &&
+         headers.Upgrade()->value() == Http::Headers::get().UpgradeValues.ConnectUdp;
+}
+
 bool HeaderUtility::isConnectResponse(const RequestHeaderMap* request_headers,
                                       const ResponseHeaderMap& response_headers) {
   return request_headers && isConnect(*request_headers) &&
          static_cast<Http::Code>(Http::Utility::getResponseStatus(response_headers)) ==
              Http::Code::OK;
 }
 
+bool HeaderUtility::rewriteAuthorityForConnectUdp(RequestHeaderMap& headers) {
+  // Per RFC 9298, the URI template must only contain ASCII characters in the range 0x21-0x7E.
+  absl::string_view path = headers.getPathValue();
+  for (char c : path) {
+    unsigned char ascii_code = static_cast<unsigned char>(c);
+    if (ascii_code < 0x21 || ascii_code > 0x7e) {
+      ENVOY_LOG_MISC(warn, "CONNECT-UDP request with a bad character in the path {}", path);
+      return false;
+    }
+  }
+
+  // Extract target host and port from path using default template.
+  if (!absl::StartsWith(path, "/.well-known/masque/udp/")) {
+    ENVOY_LOG_MISC(warn, "CONNECT-UDP request path is not a well-known URI: {}", path);
+    return false;
+  }
+
+  std::vector<absl::string_view> path_split = absl::StrSplit(path, '/');
+  if (path_split.size() != 7 || path_split[4].empty() || path_split[5].empty() ||
+      !path_split[6].empty()) {
+    ENVOY_LOG_MISC(warn, "CONNECT-UDP request with a malformed URI template in the path {}", path);
+    return false;
+  }
+
+  // Utility::PercentEncoding::decode never returns an empty string if the input argument is not
+  // empty.
+  std::string target_host = Utility::PercentEncoding::decode(path_split[4]);
+  // Per RFC 9298, IPv6 Zone ID is not supported.
+  if (target_host.find('%') != std::string::npos) {
+    ENVOY_LOG_MISC(warn, "CONNECT-UDP request with a non-escpaed char (%) in the path {}", path);
+    return false;
+  }
+  std::string target_port = Utility::PercentEncoding::decode(path_split[5]);
+
+  // If the host is an IPv6 address, surround the address with square brackets.
+  in6_addr sin6_addr;
+  bool is_ipv6 = (inet_pton(AF_INET6, target_host.c_str(), &sin6_addr) == 1);
+  std::string new_host =
+      absl::StrCat((is_ipv6 ? absl::StrCat("[", target_host, "]") : target_host), ":", target_port);
+  headers.setHost(new_host);
+
+  return true;
+}
+
 #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
 bool HeaderUtility::isCapsuleProtocol(const RequestOrResponseHeaderMap& headers) {
   Http::HeaderMap::GetResult capsule_protocol =

diff --git a/source/common/http/header_utility.h b/source/common/http/header_utility.h
@@ -171,12 +171,23 @@ class HeaderUtility {
    */
   static bool isConnect(const RequestHeaderMap& headers);
 
+  /**
+   * @brief a helper function to determine if the headers represent a CONNECT-UDP request.
+   */
+  static bool isConnectUdp(const RequestHeaderMap& headers);
+
   /**
    * @brief a helper function to determine if the headers represent an accepted CONNECT response.
    */
   static bool isConnectResponse(const RequestHeaderMap* request_headers,
                                 const ResponseHeaderMap& response_headers);
 
+  /**
+   * @brief Rewrites the authority header field by parsing the path using the default CONNECT-UDP
+   * URI template. Returns true if the parsing was successful, otherwise returns false.
+   */
+  static bool rewriteAuthorityForConnectUdp(RequestHeaderMap& headers);
+
 #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
   /**
    * @brief Returns true if the Capsule-Protocol header field (RFC 9297) is set to true. If the

diff --git a/source/common/http/headers.h b/source/common/http/headers.h
@@ -142,6 +142,7 @@ class HeaderValues {
 
   const LowerCaseString ProxyAuthenticate{"proxy-authenticate"};
   const LowerCaseString ProxyAuthorization{"proxy-authorization"};
+  const LowerCaseString CapsuleProtocol{"capsule-protocol"};
   const LowerCaseString ClientTraceId{"x-client-trace-id"};
   const LowerCaseString Connection{"connection"};
   const LowerCaseString ContentLength{"content-length"};
@@ -246,6 +247,7 @@ class HeaderValues {
   struct {
     const std::string H2c{"h2c"};
     const std::string WebSocket{"websocket"};
+    const std::string ConnectUdp{"connect-udp"};
   } UpgradeValues;
 
   struct {

diff --git a/source/common/network/connection_impl_base.h b/source/common/network/connection_impl_base.h
@@ -23,7 +23,7 @@ class ConnectionImplBase : public FilterManagerConnection,
   // Network::Connection
   void addConnectionCallbacks(ConnectionCallbacks& cb) override;
   void removeConnectionCallbacks(ConnectionCallbacks& cb) override;
-  Event::Dispatcher& dispatcher() override { return dispatcher_; }
+  Event::Dispatcher& dispatcher() const override { return dispatcher_; }
   uint64_t id() const override { return id_; }
   void hashKey(std::vector<uint8_t>& hash) const override;
   void setConnectionStats(const ConnectionStats& stats) override;

diff --git a/source/common/network/multi_connection_base_impl.cc b/source/common/network/multi_connection_base_impl.cc
@@ -346,7 +346,7 @@ void MultiConnectionBaseImpl::close(ConnectionCloseType type, absl::string_view
   connections_[0]->close(type, details);
 }
 
-Event::Dispatcher& MultiConnectionBaseImpl::dispatcher() {
+Event::Dispatcher& MultiConnectionBaseImpl::dispatcher() const {
   ASSERT(&dispatcher_ == &connections_[0]->dispatcher());
   return connections_[0]->dispatcher();
 }

diff --git a/source/common/network/multi_connection_base_impl.h b/source/common/network/multi_connection_base_impl.h
@@ -124,7 +124,7 @@ class MultiConnectionBaseImpl : public ClientConnection,
 
   // Methods implemented largely by this class itself.
   uint64_t id() const override;
-  Event::Dispatcher& dispatcher() override;
+  Event::Dispatcher& dispatcher() const override;
   void close(ConnectionCloseType type) override { close(type, ""); }
   void close(ConnectionCloseType type, absl::string_view details) override;
   bool readEnabled() const override;

diff --git a/source/common/quic/envoy_quic_client_session.h b/source/common/quic/envoy_quic_client_session.h
@@ -65,9 +65,11 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl,
                                    std::unique_ptr<quic::QuicEncrypter> encrypter) override;
 
   quic::HttpDatagramSupport LocalHttpDatagramSupport() override {
-    // TODO(https://github.com/envoyproxy/envoy/issues/23564): Http3 Datagram support should be
-    // turned on by returning quic::HttpDatagramSupport::kRfc once the CONNECT-UDP support work is
-    // completed.
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+    if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support")) {
+      return quic::HttpDatagramSupport::kRfc;
+    }
+#endif
     return quic::HttpDatagramSupport::kNone;
   }
 

diff --git a/source/common/quic/envoy_quic_client_stream.cc b/source/common/quic/envoy_quic_client_stream.cc
@@ -87,6 +87,13 @@ Http::Status EnvoyQuicClientStream::encodeHeaders(const Http::RequestHeaderMap&
   if (headers.Method()->value() == "HEAD") {
     sent_head_request_ = true;
   }
+#endif
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+  if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") &&
+      (Http::HeaderUtility::isCapsuleProtocol(headers) ||
+       Http::HeaderUtility::isConnectUdp(headers))) {
+    useCapsuleProtocol();
+  }
 #endif
   {
     IncrementalBytesSentTracker tracker(*this, *mutableBytesMeter(), true);

diff --git a/source/common/quic/envoy_quic_client_stream.h b/source/common/quic/envoy_quic_client_stream.h
@@ -51,12 +51,6 @@ class EnvoyQuicClientStream : public quic::QuicSpdyClientStream,
   void OnConnectionClosed(quic::QuicErrorCode error, quic::ConnectionCloseSource source) override;
 
   void clearWatermarkBuffer();
-#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
-  // Makes the QUIC stream use Capsule Protocol. Once this method is called, any calls to encodeData
-  // are expected to contain capsules which will be sent along as HTTP Datagrams. Also, the stream
-  // starts to receive HTTP/3 Datagrams and decode into Capsules.
-  void useCapsuleProtocol();
-#endif
 
 protected:
   // EnvoyQuicStream
@@ -83,6 +77,13 @@ class EnvoyQuicClientStream : public quic::QuicSpdyClientStream,
   // Deliver awaiting trailers if body has been delivered.
   void maybeDecodeTrailers();
 
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+  // Makes the QUIC stream use Capsule Protocol. Once this method is called, any calls to encodeData
+  // are expected to contain capsules which will be sent along as HTTP Datagrams. Also, the stream
+  // starts to receive HTTP/3 Datagrams and decode into Capsules.
+  void useCapsuleProtocol();
+#endif
+
   Http::ResponseDecoder* response_decoder_{nullptr};
   bool decoded_1xx_{false};
 #ifdef ENVOY_ENABLE_HTTP_DATAGRAMS

diff --git a/source/common/quic/envoy_quic_server_session.h b/source/common/quic/envoy_quic_server_session.h
@@ -115,9 +115,11 @@ class EnvoyQuicServerSession : public quic::QuicServerSessionBase,
   quic::QuicSpdyStream* CreateOutgoingUnidirectionalStream() override;
 
   quic::HttpDatagramSupport LocalHttpDatagramSupport() override {
-    // TODO(jeongseokson): Http3 Datagram support should be turned on by returning
-    // quic::HttpDatagramSupport::kRfc once the CONNECT-UDP support work is completed.
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+    return quic::HttpDatagramSupport::kRfc;
+#else
     return quic::HttpDatagramSupport::kNone;
+#endif
   }
 
   // QuicFilterManagerConnectionImpl

diff --git a/source/common/quic/envoy_quic_server_stream.cc b/source/common/quic/envoy_quic_server_stream.cc
@@ -41,6 +41,11 @@ EnvoyQuicServerStream::EnvoyQuicServerStream(
 
   stats_gatherer_ = new QuicStatsGatherer(&filterManagerConnection()->dispatcher().timeSource());
   set_ack_listener(stats_gatherer_);
+  // TODO(https://github.com/envoyproxy/envoy/issues/23564): Remove this line when the QUICHE is
+  // updated with a more reasonable default expiry time for QUIC Datagrams.
+  if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support")) {
+    SetMaxDatagramTimeInQueue(::quic::QuicTime::Delta::FromMilliseconds(100));
+  }
 }
 
 void EnvoyQuicServerStream::encode1xxHeaders(const Http::ResponseHeaderMap& headers) {
@@ -261,6 +266,14 @@ void EnvoyQuicServerStream::OnInitialHeadersComplete(bool fin, size_t frame_len,
   }
 #endif
 
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+  if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.enable_connect_udp_support") &&
+      (Http::HeaderUtility::isCapsuleProtocol(*headers) ||
+       Http::HeaderUtility::isConnectUdp(*headers))) {
+    useCapsuleProtocol();
+  }
+#endif
+
   request_decoder_->decodeHeaders(std::move(headers), /*end_stream=*/fin);
   ConsumeHeaderList();
 }

diff --git a/source/common/quic/envoy_quic_server_stream.h b/source/common/quic/envoy_quic_server_stream.h
@@ -79,13 +79,6 @@ class EnvoyQuicServerStream : public quic::QuicSpdyServerStreamBase,
   Http::HeaderUtility::HeaderValidationResult
   validateHeader(absl::string_view header_name, absl::string_view header_value) override;
 
-#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
-  // Makes the QUIC stream use Capsule Protocol. Once this method is called, any calls to encodeData
-  // are expected to contain capsules which will be sent along as HTTP Datagrams. Also, the stream
-  // starts to receive HTTP/3 Datagrams and decode into Capsules.
-  void useCapsuleProtocol();
-#endif
-
 protected:
   // EnvoyQuicStream
   void switchStreamBlockState() override;
@@ -113,6 +106,13 @@ class EnvoyQuicServerStream : public quic::QuicSpdyServerStreamBase,
   // Deliver awaiting trailers if body has been delivered.
   void maybeDecodeTrailers();
 
+#ifdef ENVOY_ENABLE_HTTP_DATAGRAMS
+  // Makes the QUIC stream use Capsule Protocol. Once this method is called, any calls to encodeData
+  // are expected to contain capsules which will be sent along as HTTP Datagrams. Also, the stream
+  // starts to receive HTTP/3 Datagrams and decode into Capsules.
+  void useCapsuleProtocol();
+#endif
+
   Http::RequestDecoder* request_decoder_{nullptr};
   envoy::config::core::v3::HttpProtocolOptions::HeadersWithUnderscoresAction
       headers_with_underscores_action_;