upstream: Implement retry concurrency budgets

api/envoy/api/v2/cluster/BUILD

@@ -5,5 +5,8 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
 licenses(["notice"])  # Apache 2
 
 api_proto_package(
-    deps = ["//envoy/api/v2/core:pkg"],
+    deps = [
+        "//envoy/api/v2/core:pkg",
+        "//envoy/type:pkg",
+    ],
 )

api/envoy/api/v2/cluster/circuit_breaker.proto

@@ -9,6 +9,7 @@ option csharp_namespace = "Envoy.Api.V2.ClusterNS";
 option ruby_package = "Envoy.Api.V2.ClusterNS";
 
 import "envoy/api/v2/core/base.proto";
+import "envoy/type/percent.proto";
 
 import "google/protobuf/wrappers.proto";
 
@@ -21,8 +22,23 @@ import "validate/validate.proto";
 message CircuitBreakers {
   // A Thresholds defines CircuitBreaker settings for a
   // :ref:`RoutingPriority<envoy_api_enum_core.RoutingPriority>`.
-  // [#next-free-field: 8]
+  // [#next-free-field: 9]
   message Thresholds {
+    message RetryBudget {
+      // Specifies the limit on concurrent retries as a percentage of the sum of active requests and
+      // active pending requests. For example, if there are 100 active requests and the
+      // budget_percent is set to 25, there may be 25 active retries.
+      //
+      // This parameter is optional. Defaults to 20%.
+      type.Percent budget_percent = 1;
+
+      // Specifies the minimum retry concurrency allowed for the retry budget. The limit on the
+      // number of active retries may never go below this number.
+      //
+      // This parameter is optional. Defaults to 3.
+      google.protobuf.UInt32Value min_retry_concurrency = 2;
+    }
+
     // The :ref:`RoutingPriority<envoy_api_enum_core.RoutingPriority>`
     // the specified CircuitBreaker settings apply to.
     core.RoutingPriority priority = 1 [(validate.rules).enum = {defined_only: true}];
@@ -39,9 +55,22 @@ message CircuitBreakers {
     // upstream cluster. If not specified, the default is 1024.
     google.protobuf.UInt32Value max_requests = 4;
 
-    // The maximum number of parallel retries that Envoy will allow to the
-    // upstream cluster. If not specified, the default is 3.
-    google.protobuf.UInt32Value max_retries = 5;
+    // If neither max_retries or retry_budget is specified, a max_retries value
+    // of 3 is enforced.
+    oneof retry_threshold {
+      // The maximum number of parallel retries that Envoy will allow to the
+      // upstream cluster. If not specified, the default is 3.
+      google.protobuf.UInt32Value max_retries = 5;
+
+      // Specifies a limit on concurrent retries in relation to the number of active requests. This
+      // parameter is optional.
+      //
+      // .. note::
+      //
+      //    If this field is set, the retry budget will override any configured retry circuit
+      //    breaker.
+      RetryBudget retry_budget = 8;
+    }
 
     // If track_remaining is true, then stats will be published that expose
     // the number of resources remaining until the circuit breakers open. If

api/envoy/api/v3alpha/cluster/BUILD

@@ -8,6 +8,7 @@ api_proto_package(
     deps = [
         "//envoy/api/v2/cluster:pkg",
         "//envoy/api/v3alpha/core:pkg",
+        "//envoy/type/v3alpha:pkg",
         "@com_github_cncf_udpa//udpa/annotations:pkg",
     ],
 )

api/envoy/api/v3alpha/cluster/circuit_breaker.proto

@@ -7,6 +7,7 @@ option java_outer_classname = "CircuitBreakerProto";
 option java_multiple_files = true;
 
 import "envoy/api/v3alpha/core/base.proto";
+import "envoy/type/v3alpha/percent.proto";
 
 import "google/protobuf/wrappers.proto";
 
@@ -24,11 +25,29 @@ message CircuitBreakers {
 
   // A Thresholds defines CircuitBreaker settings for a
   // :ref:`RoutingPriority<envoy_api_enum_api.v3alpha.core.RoutingPriority>`.
-  // [#next-free-field: 8]
+  // [#next-free-field: 9]
   message Thresholds {
     option (udpa.annotations.versioning).previous_message_type =
         "envoy.api.v2.cluster.CircuitBreakers.Thresholds";
 
+    message RetryBudget {
+      option (udpa.annotations.versioning).previous_message_type =
+          "envoy.api.v2.cluster.CircuitBreakers.Thresholds.RetryBudget";
+
+      // Specifies the limit on concurrent retries as a percentage of the sum of active requests and
+      // active pending requests. For example, if there are 100 active requests and the
+      // budget_percent is set to 25, there may be 25 active retries.
+      //
+      // This parameter is optional. Defaults to 20%.
+      type.v3alpha.Percent budget_percent = 1;
+
+      // Specifies the minimum retry concurrency allowed for the retry budget. The limit on the
+      // number of active retries may never go below this number.
+      //
+      // This parameter is optional. Defaults to 3.
+      google.protobuf.UInt32Value min_retry_concurrency = 2;
+    }
+
     // The :ref:`RoutingPriority<envoy_api_enum_api.v3alpha.core.RoutingPriority>`
     // the specified CircuitBreaker settings apply to.
     core.RoutingPriority priority = 1 [(validate.rules).enum = {defined_only: true}];
@@ -45,9 +64,22 @@ message CircuitBreakers {
     // upstream cluster. If not specified, the default is 1024.
     google.protobuf.UInt32Value max_requests = 4;
 
-    // The maximum number of parallel retries that Envoy will allow to the
-    // upstream cluster. If not specified, the default is 3.
-    google.protobuf.UInt32Value max_retries = 5;
+    // If neither max_retries or retry_budget is specified, a max_retries value
+    // of 3 is enforced.
+    oneof retry_threshold {
+      // The maximum number of parallel retries that Envoy will allow to the
+      // upstream cluster. If not specified, the default is 3.
+      google.protobuf.UInt32Value max_retries = 5;
+
+      // Specifies a limit on concurrent retries in relation to the number of active requests. This
+      // parameter is optional.
+      //
+      // .. note::
+      //
+      //    If this field is set, the retry budget will override any configured retry circuit
+      //    breaker.
+      RetryBudget retry_budget = 8;
+    }
 
     // If track_remaining is true, then stats will be published that expose
     // the number of resources remaining until the circuit breakers open. If

docs/root/configuration/upstream/cluster_manager/cluster_stats.rst

@@ -74,7 +74,7 @@ Every cluster has a statistics tree rooted at *cluster.<name>.* with the followi
   upstream_rq_tx_reset, Counter, Total requests that were reset locally
   upstream_rq_retry, Counter, Total request retries
   upstream_rq_retry_success, Counter, Total request retry successes
-  upstream_rq_retry_overflow, Counter, Total requests not retried due to circuit breaking
+  upstream_rq_retry_overflow, Counter, Total requests not retried due to circuit breaking or exceeding the retry budget
   upstream_flow_control_paused_reading_total, Counter, Total number of times flow control paused reading from upstream
   upstream_flow_control_resumed_reading_total, Counter, Total number of times flow control resumed reading from upstream
   upstream_flow_control_backed_up_total, Counter, Total number of times the upstream connection backed up and paused reads from downstream

docs/root/intro/arch_overview/http/http_routing.rst

@@ -106,6 +106,8 @@ headers <config_http_filters_router_headers_consumed>`. The following configurat
 * **Retry conditions**: Envoy can retry on different types of conditions depending on application
   requirements. For example, network failure, all 5xx response codes, idempotent 4xx response codes,
   etc.
+* **Retry budgets**: Envoy can limit the proportion of active requests via :ref:`retry budgets <envoy_api_field_cluster.CircuitBreakers.Thresholds.retry_budget>` that can be retries to
+  prevent their contribution to large increases in traffic volume.
 * **Host selection retry plugins**: Envoy can be configured to apply additional logic to the host
   selection logic when selecting hosts for retries. Specifying a
   :ref:`retry host predicate <envoy_api_field_route.RetryPolicy.retry_host_predicate>`

docs/root/intro/arch_overview/upstream/circuit_breaking.rst

@@ -30,8 +30,8 @@ configure and code each application independently. Envoy supports various types
   overflows the :ref:`upstream_rq_pending_overflow <config_cluster_manager_cluster_stats>` counter
   for the cluster will increment.
 * **Cluster maximum active retries**: The maximum number of retries that can be outstanding to all
-  hosts in a cluster at any given time. In general we recommend aggressively circuit breaking
-  retries so that retries for sporadic failures are allowed but the overall retry volume cannot
+  hosts in a cluster at any given time. In general we recommend using :ref:`retry budgets <envoy_api_field_cluster.CircuitBreakers.Thresholds.retry_budget>`; however, if static circuit breaking is preferred it should aggressively circuit break
+  retries. This is so that retries for sporadic failures are allowed, but the overall retry volume cannot
   explode and cause large scale cascading failure. If this circuit breaker overflows the
   :ref:`upstream_rq_retry_overflow <config_cluster_manager_cluster_stats>` counter for the cluster
   will increment.

docs/root/intro/version_history.rst

@@ -20,6 +20,7 @@ Version history
 * rbac: added support for matching all subject alt names instead of first in :ref:`principal_name <envoy_api_field_config.rbac.v2.Principal.Authenticated.principal_name>`.
 * redis: performance improvement for larger split commands by avoiding string copies.
 * redis: correctly follow MOVE/ASK redirection for mirrored clusters.
+* router: added support for percentage-based :ref:`retry budgets <envoy_api_field_cluster.CircuitBreakers.Thresholds.retry_budget>`
 * router: added support for REQ(header-name) :ref:`header formatter <config_http_conn_man_headers_custom_request_headers>`.
 * router: allow using a :ref:`query parameter
   <envoy_api_field_route.RouteAction.HashPolicy.query_parameter>` for HTTP consistent hashing.

include/envoy/router/router.h

@@ -246,6 +246,16 @@ class RetryState {
 public:
   using DoRetryCallback = std::function<void()>;
 
+  /**
+   * RetryBudgetStatus whether the retry budget has been configured or allows for additional
+   * retries.
+   */
+  enum class RetryBudgetStatus {
+    Unconfigured,
+    Available,
+    Exceeded,
+  };
+
   virtual ~RetryState() = default;
 
   /**

include/envoy/upstream/resource_manager.h

@@ -47,6 +47,11 @@ class Resource {
    * @return the current maximum allowed number of this resource.
    */
   virtual uint64_t max() PURE;
+
+  /**
+   * @return the current resource count.
+   */
+  virtual uint64_t count() PURE;
 };
 
 /**

include/envoy/upstream/upstream.h

@@ -664,6 +664,14 @@ class ClusterInfo {
     static const uint64_t CLOSE_CONNECTIONS_ON_HOST_HEALTH_FAILURE = 0x4;
   };
 
+  struct RetryBudget {
+    // The percentage of outstanding requests that are allowed to be retries.
+    double budget_percent;
+
+    // The minimum retry concurrency allowed to be enforced by the retry budget.
+    uint32_t min_retry_concurrency;
+  };
+
   virtual ~ClusterInfo() = default;
 
   /**
@@ -869,6 +877,11 @@ class ClusterInfo {
   virtual Http::Protocol
   upstreamHttpProtocol(absl::optional<Http::Protocol> downstream_protocol) const PURE;
 
+  /**
+   * @return the retry budget associated with a priority for the cluster if configured.
+   */
+  virtual absl::optional<RetryBudget> retryBudget(ResourcePriority priority) const PURE;
+
 protected:
   /**
    * Invoked by extensionProtocolOptionsTyped.

source/common/http/http1/conn_pool.cc

@@ -267,11 +267,20 @@ ConnPoolImpl::StreamWrapper::StreamWrapper(StreamDecoder& response_decoder, Acti
   StreamEncoderWrapper::inner_.getStream().addCallbacks(*this);
   parent_.parent_.host_->cluster().stats().upstream_rq_active_.inc();
   parent_.parent_.host_->stats().rq_active_.inc();
+
+  // TODO (tonya11en): At the time of writing, there is no way to mix different versions of HTTP
+  // traffic in the same cluster, so incrementing the request count in the per-cluster resource
+  // manager will not affect circuit breaking in any unexpected ways. Ideally, outstanding requests
+  // counts would be tracked the same way in all HTTP versions.
+  //
+  // See: https://github.com/envoyproxy/envoy/issues/9215
+  parent_.parent_.host_->cluster().resourceManager(parent_.parent_.priority_).requests().inc();
 }
 
 ConnPoolImpl::StreamWrapper::~StreamWrapper() {
   parent_.parent_.host_->cluster().stats().upstream_rq_active_.dec();
   parent_.parent_.host_->stats().rq_active_.dec();
+  parent_.parent_.host_->cluster().resourceManager(parent_.parent_.priority_).requests().dec();
 }
 
 void ConnPoolImpl::StreamWrapper::onEncodeComplete() { encode_complete_ = true; }

source/common/router/retry_state_impl.cc

@@ -223,7 +223,25 @@ RetryStatus RetryStateImpl::shouldRetry(bool would_retry, DoRetryCallback callba
 
   retries_remaining_--;
 
-  if (!cluster_.resourceManager(priority_).retries().canCreate()) {
+  // Only consider the max_retries circuit breaker if a retry budget is not configured. Otherwise,
+  // only consider the retry budget.
+  const RetryBudgetStatus budget_status = retryBudgetStatus(priority_);
+  bool retry_overflow;
+  switch (budget_status) {
+  case RetryBudgetStatus::Exceeded:
+    retry_overflow = true;
+    break;
+  case RetryBudgetStatus::Unconfigured:
+    retry_overflow = !cluster_.resourceManager(priority_).retries().canCreate();
+    break;
+  case RetryBudgetStatus::Available:
+    retry_overflow = false;
+    break;
+  default:
+    NOT_IMPLEMENTED_GCOVR_EXCL_LINE;
+  }
+
+  if (retry_overflow) {
     cluster_.stats().upstream_rq_retry_overflow_.inc();
     return RetryStatus::NoOverflow;
   }
@@ -360,5 +378,24 @@ bool RetryStateImpl::wouldRetryFromReset(const Http::StreamResetReason reset_rea
   return false;
 }
 
+RetryStateImpl::RetryBudgetStatus
+RetryStateImpl::retryBudgetStatus(Upstream::ResourcePriority priority) const {
+  const auto retry_budget = cluster_.retryBudget(priority);
+  if (!retry_budget) {
+    return RetryBudgetStatus::Unconfigured;
+  }
+
+  const uint64_t current_active = cluster_.resourceManager(priority).requests().count() +
+                                  cluster_.resourceManager(priority).pendingRequests().count();
+
+  const double budget = std::max<uint64_t>(current_active * retry_budget->budget_percent / 100.0,
+                                           retry_budget->min_retry_concurrency);
+
+  if (cluster_.resourceManager(priority).retries().count() >= budget) {
+    return RetryBudgetStatus::Exceeded;
+  }
+  return RetryBudgetStatus::Available;
+}
+
 } // namespace Router
 } // namespace Envoy

source/common/router/retry_state_impl.h

@@ -94,6 +94,7 @@ class RetryStateImpl : public RetryState {
   void resetRetry();
   bool wouldRetryFromReset(const Http::StreamResetReason reset_reason);
   RetryStatus shouldRetry(bool would_retry, DoRetryCallback callback);
+  RetryBudgetStatus retryBudgetStatus(Upstream::ResourcePriority priority) const;
 
   const Upstream::ClusterInfo& cluster_;
   Runtime::Loader& runtime_;

source/common/upstream/resource_manager_impl.h

@@ -72,6 +72,7 @@ class ResourceManagerImpl : public ResourceManager {
       open_gauge_.set(canCreate() ? 0 : 1);
     }
     uint64_t max() override { return runtime_.snapshot().getInteger(runtime_key_, max_); }
+    uint64_t count() override { return current_.load(); }
 
     /**
      * We set the gauge instead of incrementing and decrementing because,

source/common/upstream/upstream_impl.cc

@@ -29,6 +29,7 @@
 #include "common/network/socket_option_factory.h"
 #include "common/protobuf/protobuf.h"
 #include "common/protobuf/utility.h"
+#include "common/router/config_utility.h"
 #include "common/runtime/runtime_impl.h"
 #include "common/upstream/eds.h"
 #include "common/upstream/health_checker_impl.h"
@@ -759,6 +760,22 @@ ClusterInfoImpl::ClusterInfoImpl(
         factory.createFilterFactoryFromProto(*message, *factory_context_);
     filter_factories_.push_back(callback);
   }
+
+  if (config.has_circuit_breakers()) {
+    for (const auto& threshold : config.circuit_breakers().thresholds()) {
+      if (threshold.has_retry_budget()) {
+        const ResourcePriority priority =
+            Router::ConfigUtility::parsePriority(threshold.priority());
+        retry_budget_map_.emplace(priority,
+                                  RetryBudget{
+                                      .budget_percent = PROTOBUF_GET_WRAPPED_OR_DEFAULT(
+                                          threshold.retry_budget(), budget_percent, 20.0),
+                                      .min_retry_concurrency = PROTOBUF_GET_WRAPPED_OR_DEFAULT(
+                                          threshold.retry_budget(), min_retry_concurrency, 3),
+                                  });
+      }
+    }
+  }
 }
 
 ProtocolOptionsConfigConstSharedPtr

source/common/upstream/upstream_impl.h

@@ -576,6 +576,12 @@ class ClusterInfoImpl : public ClusterInfo, protected Logger::Loggable<Logger::I
   void createNetworkFilterChain(Network::Connection&) const override;
   Http::Protocol
   upstreamHttpProtocol(absl::optional<Http::Protocol> downstream_protocol) const override;
+  absl::optional<RetryBudget> retryBudget(ResourcePriority priority) const override {
+    if (!retry_budget_map_.contains(priority)) {
+      return absl::nullopt;
+    }
+    return retry_budget_map_.at(priority);
+  }
 
 private:
   struct ResourceManagers {
@@ -626,6 +632,7 @@ class ClusterInfoImpl : public ClusterInfo, protected Logger::Loggable<Logger::I
   const absl::optional<envoy::api::v2::Cluster::CustomClusterType> cluster_type_;
   const std::unique_ptr<Server::Configuration::CommonFactoryContext> factory_context_;
   std::vector<Network::FilterFactoryCb> filter_factories_;
+  absl::flat_hash_map<ResourcePriority, absl::optional<RetryBudget>> retry_budget_map_;
 };
 
 /**

test/common/http/http1/conn_pool_test.cc

@@ -150,6 +150,8 @@ struct ActiveTestRequest {
 
   ActiveTestRequest(Http1ConnPoolImplTest& parent, size_t client_index, Type type)
       : parent_(parent), client_index_(client_index) {
+    uint64_t active_rq_observed =
+        parent_.cluster_->resourceManager(Upstream::ResourcePriority::Default).requests().count();
     uint64_t current_rq_total = parent_.cluster_->stats_.upstream_rq_total_.value();
     if (type == Type::CreateConnection) {
       parent.conn_pool_.expectClientCreate();
@@ -175,6 +177,10 @@ struct ActiveTestRequest {
     }
     if (type != Type::Pending) {
       EXPECT_EQ(current_rq_total + 1, parent_.cluster_->stats_.upstream_rq_total_.value());
+      EXPECT_EQ(active_rq_observed + 1,
+                parent_.cluster_->resourceManager(Upstream::ResourcePriority::Default)
+                    .requests()
+                    .count());
     }
   }