dns cache: add dns failure refresh rate

api/envoy/config/common/dynamic_forward_proxy/v2alpha/dns_cache.proto

@@ -20,7 +20,7 @@ option (udpa.annotations.file_migrate).move_to_package =
 
 // Configuration for the dynamic forward proxy DNS cache. See the :ref:`architecture overview
 // <arch_overview_http_dynamic_forward_proxy>` for more information.
-// [#next-free-field: 6]
+// [#next-free-field: 7]
 message DnsCacheConfig {
   // The name of the cache. Multiple named caches allow independent dynamic forward proxy
   // configurations to operate within a single Envoy process using different configurations. All
@@ -70,4 +70,9 @@ message DnsCacheConfig {
   //   it is possible for the maximum hosts in the cache to go slightly above the configured
   //   value depending on timing. This is similar to how other circuit breakers work.
   google.protobuf.UInt32Value max_hosts = 5 [(validate.rules).uint32 = {gt: 0}];
+
+  // If the DNS failure refresh rate is specified,
+  // this is used as the cache's DNS refresh rate when DNS requests are failing. If this setting is
+  // not specified, the failure refresh rate defaults to the dns_refresh_rate.
+  api.v2.Cluster.RefreshRate dns_failure_refresh_rate = 6;
 }

api/envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.proto

@@ -19,7 +19,7 @@ option java_multiple_files = true;
 
 // Configuration for the dynamic forward proxy DNS cache. See the :ref:`architecture overview
 // <arch_overview_http_dynamic_forward_proxy>` for more information.
-// [#next-free-field: 6]
+// [#next-free-field: 7]
 message DnsCacheConfig {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.common.dynamic_forward_proxy.v2alpha.DnsCacheConfig";
@@ -72,4 +72,9 @@ message DnsCacheConfig {
   //   it is possible for the maximum hosts in the cache to go slightly above the configured
   //   value depending on timing. This is similar to how other circuit breakers work.
   google.protobuf.UInt32Value max_hosts = 5 [(validate.rules).uint32 = {gt: 0}];
+
+  // If the DNS failure refresh rate is specified,
+  // this is used as the cache's DNS refresh rate when DNS requests are failing. If this setting is
+  // not specified, the failure refresh rate defaults to the dns_refresh_rate.
+  config.cluster.v3.Cluster.RefreshRate dns_failure_refresh_rate = 6;
 }

docs/root/intro/version_history.rst

@@ -9,6 +9,7 @@ Version history
   minimum.
 * config: use type URL to select an extension whenever the config type URL (or its previous versions) uniquely identify a typed extension, see :ref:`extension configuration <config_overview_extension_configuration>`.
 * dns: the STRICT_DNS cluster now only resolves to 0 hosts if DNS resolution successfully returns 0 hosts.
+* dns: added support for :ref:`dns_failure_refresh_rate <envoy_api_field_config.common.dynamic_forward_proxy.v2alpha.DnsCacheConfig.dns_failure_refresh_rate>` for the :ref:`dns cache <envoy_api_msg_config.common.dynamic_forward_proxy.v2alpha.DnsCacheConfig>` to set the DNS refresh rate during failures.
 * http filters: http filter extensions use the "envoy.filters.http" name space. A mapping
   of extension names is available in the :ref:`deprecated <deprecated>` documentation.
 * ext_authz: disabled the use of lowercase string matcher for headers matching in HTTP-based `ext_authz`.

source/common/config/utility.cc

@@ -296,23 +296,5 @@ void Utility::translateOpaqueConfig(const ProtobufWkt::Any& typed_config,
   }
 }
 
-BackOffStrategyPtr
-Utility::prepareDnsRefreshStrategy(const envoy::config::cluster::v3::Cluster& cluster,
-                                   const uint64_t dns_refresh_rate_ms,
-                                   Runtime::RandomGenerator& random) {
-  if (cluster.has_dns_failure_refresh_rate()) {
-    uint64_t base_interval_ms =
-        PROTOBUF_GET_MS_REQUIRED(cluster.dns_failure_refresh_rate(), base_interval);
-    uint64_t max_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(cluster.dns_failure_refresh_rate(),
-                                                          max_interval, base_interval_ms * 10);
-    if (max_interval_ms < base_interval_ms) {
-      throw EnvoyException("cluster.dns_failure_refresh_rate must have max_interval greater than "
-                           "or equal to the base_interval");
-    }
-    return std::make_unique<JitteredBackOffStrategy>(base_interval_ms, max_interval_ms, random);
-  }
-  return std::make_unique<FixedBackOffStrategy>(dns_refresh_rate_ms);
-}
-
 } // namespace Config
 } // namespace Envoy

source/common/config/utility.h

@@ -343,14 +343,27 @@ class Utility {
 
   /**
    * Prepares the DNS failure refresh backoff strategy given the cluster configuration.
-   * @param cluster the cluster configuration.
+   * @param config the config that contains dns refresh information.
    * @param dns_refresh_rate_ms the default DNS refresh rate.
    * @param random the random generator.
    * @return BackOffStrategyPtr for scheduling refreshes.
    */
-  static BackOffStrategyPtr
-  prepareDnsRefreshStrategy(const envoy::config::cluster::v3::Cluster& cluster,
-                            uint64_t dns_refresh_rate_ms, Runtime::RandomGenerator& random);
+  template <typename T>
+  static BackOffStrategyPtr prepareDnsRefreshStrategy(const T& config, uint64_t dns_refresh_rate_ms,
+                                                      Runtime::RandomGenerator& random) {
+    if (config.has_dns_failure_refresh_rate()) {
+      uint64_t base_interval_ms =
+          PROTOBUF_GET_MS_REQUIRED(config.dns_failure_refresh_rate(), base_interval);
+      uint64_t max_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(config.dns_failure_refresh_rate(),
+                                                            max_interval, base_interval_ms * 10);
+      if (max_interval_ms < base_interval_ms) {
+        throw EnvoyException("dns_failure_refresh_rate must have max_interval greater than "
+                             "or equal to the base_interval");
+      }
+      return std::make_unique<JitteredBackOffStrategy>(base_interval_ms, max_interval_ms, random);
+    }
+    return std::make_unique<FixedBackOffStrategy>(dns_refresh_rate_ms);
+  }
 };
 
 } // namespace Config

source/common/upstream/logical_dns_cluster.cc

@@ -59,8 +59,9 @@ LogicalDnsCluster::LogicalDnsCluster(
           cluster.has_load_assignment()
               ? convertPriority(cluster.load_assignment())
               : Config::Utility::translateClusterHosts(cluster.hidden_envoy_deprecated_hosts())) {
-  failure_backoff_strategy_ = Config::Utility::prepareDnsRefreshStrategy(
-      cluster, dns_refresh_rate_ms_.count(), factory_context.random());
+  failure_backoff_strategy_ =
+      Config::Utility::prepareDnsRefreshStrategy<envoy::config::cluster::v3::Cluster>(
+          cluster, dns_refresh_rate_ms_.count(), factory_context.random());
 
   const auto& locality_lb_endpoints = load_assignment_.endpoints();
   if (locality_lb_endpoints.size() != 1 || locality_lb_endpoints[0].lb_endpoints().size() != 1) {

source/common/upstream/strict_dns_cluster.cc

@@ -19,8 +19,9 @@ StrictDnsClusterImpl::StrictDnsClusterImpl(
       dns_refresh_rate_ms_(
           std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate, 5000))),
       respect_dns_ttl_(cluster.respect_dns_ttl()) {
-  failure_backoff_strategy_ = Config::Utility::prepareDnsRefreshStrategy(
-      cluster, dns_refresh_rate_ms_.count(), factory_context.random());
+  failure_backoff_strategy_ =
+      Config::Utility::prepareDnsRefreshStrategy<envoy::config::cluster::v3::Cluster>(
+          cluster, dns_refresh_rate_ms_.count(), factory_context.random());
 
   std::list<ResolveTargetPtr> resolve_targets;
   const envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment(

source/extensions/clusters/dynamic_forward_proxy/cluster.cc

@@ -187,7 +187,8 @@ ClusterFactory::createClusterWithConfig(
     Server::Configuration::TransportSocketFactoryContextImpl& socket_factory_context,
     Stats::ScopePtr&& stats_scope) {
   Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory(
-      context.singletonManager(), context.dispatcher(), context.tls(), context.stats());
+      context.singletonManager(), context.dispatcher(), context.tls(), context.random(),
+      context.stats());
   envoy::config::cluster::v3::Cluster cluster_config = cluster;
   if (cluster_config.has_upstream_http_protocol_options()) {
     if (!cluster_config.upstream_http_protocol_options().auto_sni() ||

source/extensions/common/dynamic_forward_proxy/BUILD

@@ -12,6 +12,7 @@ envoy_cc_library(
     name = "dns_cache_interface",
     hdrs = ["dns_cache.h"],
     deps = [
+        "//include/envoy/common:backoff_strategy_interface",
         "//include/envoy/event:dispatcher_interface",
         "//include/envoy/singleton:manager_interface",
         "//include/envoy/thread_local:thread_local_interface",
@@ -39,6 +40,7 @@ envoy_cc_library(
         "//include/envoy/network:dns_interface",
         "//include/envoy/thread_local:thread_local_interface",
         "//source/common/common:cleanup_lib",
+        "//source/common/config:utility_lib",
         "//source/common/network:utility_lib",
         "//source/common/upstream:upstream_lib",
         "@envoy_api//envoy/extensions/common/dynamic_forward_proxy/v3:pkg_cc_proto",

source/extensions/common/dynamic_forward_proxy/dns_cache.h

@@ -175,7 +175,9 @@ using DnsCacheManagerSharedPtr = std::shared_ptr<DnsCacheManager>;
  */
 DnsCacheManagerSharedPtr getCacheManager(Singleton::Manager& manager,
                                          Event::Dispatcher& main_thread_dispatcher,
-                                         ThreadLocal::SlotAllocator& tls, Stats::Scope& root_scope);
+                                         ThreadLocal::SlotAllocator& tls,
+                                         Runtime::RandomGenerator& random,
+                                         Stats::Scope& root_scope);
 
 /**
  * Factory for getting a DNS cache manager.

source/extensions/common/dynamic_forward_proxy/dns_cache_impl.cc

@@ -2,6 +2,7 @@
 
 #include "envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.pb.h"
 
+#include "common/config/utility.h"
 #include "common/http/utility.h"
 #include "common/network/utility.h"
 
@@ -15,14 +16,18 @@ namespace DynamicForwardProxy {
 
 DnsCacheImpl::DnsCacheImpl(
     Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls,
-    Stats::Scope& root_scope,
+    Runtime::RandomGenerator& random, Stats::Scope& root_scope,
     const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config)
     : main_thread_dispatcher_(main_thread_dispatcher),
       dns_lookup_family_(Upstream::getDnsLookupFamilyFromEnum(config.dns_lookup_family())),
       resolver_(main_thread_dispatcher.createDnsResolver({}, false)), tls_slot_(tls.allocateSlot()),
       scope_(root_scope.createScope(fmt::format("dns_cache.{}.", config.name()))),
       stats_{ALL_DNS_CACHE_STATS(POOL_COUNTER(*scope_), POOL_GAUGE(*scope_))},
       refresh_interval_(PROTOBUF_GET_MS_OR_DEFAULT(config, dns_refresh_rate, 60000)),
+      failure_backoff_strategy_(
+          Config::Utility::prepareDnsRefreshStrategy<
+              envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig>(
+              config, refresh_interval_.count(), random)),
       host_ttl_(PROTOBUF_GET_MS_OR_DEFAULT(config, host_ttl, 300000)),
       max_hosts_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, max_hosts, 1024)) {
   tls_slot_->set([](Event::Dispatcher&) { return std::make_shared<ThreadLocalHostInfo>(); });
@@ -197,9 +202,13 @@ void DnsCacheImpl::finishResolve(const std::string& host,
   // Kick off the refresh timer.
   // TODO(mattklein123): Consider jitter here. It may not be necessary since the initial host
   // is populated dynamically.
-  // TODO(junr03): more aggressive refresh interval when DNS resolution fails.
-  // related issue: https://github.com/lyft/envoy-mobile/issues/673
-  primary_host_info.refresh_timer_->enableTimer(refresh_interval_);
+  if (status == Network::DnsResolver::ResolutionStatus::Success) {
+    failure_backoff_strategy_->reset();
+    primary_host_info.refresh_timer_->enableTimer(refresh_interval_);
+  } else {
+    primary_host_info.refresh_timer_->enableTimer(
+        std::chrono::milliseconds(failure_backoff_strategy_->nextBackOffMs()));
+  }
 }
 
 void DnsCacheImpl::runAddUpdateCallbacks(const std::string& host,

source/extensions/common/dynamic_forward_proxy/dns_cache_impl.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "envoy/common/backoff_strategy.h"
 #include "envoy/extensions/common/dynamic_forward_proxy/v3/dns_cache.pb.h"
 #include "envoy/network/dns.h"
 #include "envoy/thread_local/thread_local.h"
@@ -38,7 +39,7 @@ struct DnsCacheStats {
 class DnsCacheImpl : public DnsCache, Logger::Loggable<Logger::Id::forward_proxy> {
 public:
   DnsCacheImpl(Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls,
-               Stats::Scope& root_scope,
+               Runtime::RandomGenerator& random, Stats::Scope& root_scope,
                const envoy::extensions::common::dynamic_forward_proxy::v3::DnsCacheConfig& config);
   ~DnsCacheImpl() override;
 
@@ -138,6 +139,7 @@ class DnsCacheImpl : public DnsCache, Logger::Loggable<Logger::Id::forward_proxy
   std::list<AddUpdateCallbacksHandleImpl*> update_callbacks_;
   absl::flat_hash_map<std::string, PrimaryHostInfoPtr> primary_hosts_;
   const std::chrono::milliseconds refresh_interval_;
+  const BackOffStrategyPtr failure_backoff_strategy_;
   const std::chrono::milliseconds host_ttl_;
   const uint32_t max_hosts_;
 };

source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.cc

@@ -28,19 +28,21 @@ DnsCacheSharedPtr DnsCacheManagerImpl::getCache(
   }
 
   DnsCacheSharedPtr new_cache =
-      std::make_shared<DnsCacheImpl>(main_thread_dispatcher_, tls_, root_scope_, config);
+      std::make_shared<DnsCacheImpl>(main_thread_dispatcher_, tls_, random_, root_scope_, config);
   caches_.emplace(config.name(), ActiveCache{config, new_cache});
   return new_cache;
 }
 
 DnsCacheManagerSharedPtr getCacheManager(Singleton::Manager& singleton_manager,
                                          Event::Dispatcher& main_thread_dispatcher,
                                          ThreadLocal::SlotAllocator& tls,
+                                         Runtime::RandomGenerator& random,
                                          Stats::Scope& root_scope) {
   return singleton_manager.getTyped<DnsCacheManager>(
       SINGLETON_MANAGER_REGISTERED_NAME(dns_cache_manager),
-      [&main_thread_dispatcher, &tls, &root_scope] {
-        return std::make_shared<DnsCacheManagerImpl>(main_thread_dispatcher, tls, root_scope);
+      [&main_thread_dispatcher, &tls, &random, &root_scope] {
+        return std::make_shared<DnsCacheManagerImpl>(main_thread_dispatcher, tls, random,
+                                                     root_scope);
       });
 }
 

source/extensions/common/dynamic_forward_proxy/dns_cache_manager_impl.h

@@ -14,8 +14,9 @@ namespace DynamicForwardProxy {
 class DnsCacheManagerImpl : public DnsCacheManager, public Singleton::Instance {
 public:
   DnsCacheManagerImpl(Event::Dispatcher& main_thread_dispatcher, ThreadLocal::SlotAllocator& tls,
-                      Stats::Scope& root_scope)
-      : main_thread_dispatcher_(main_thread_dispatcher), tls_(tls), root_scope_(root_scope) {}
+                      Runtime::RandomGenerator& random, Stats::Scope& root_scope)
+      : main_thread_dispatcher_(main_thread_dispatcher), tls_(tls), random_(random),
+        root_scope_(root_scope) {}
 
   // DnsCacheManager
   DnsCacheSharedPtr getCache(
@@ -33,25 +34,28 @@ class DnsCacheManagerImpl : public DnsCacheManager, public Singleton::Instance {
 
   Event::Dispatcher& main_thread_dispatcher_;
   ThreadLocal::SlotAllocator& tls_;
+  Runtime::RandomGenerator& random_;
   Stats::Scope& root_scope_;
   absl::flat_hash_map<std::string, ActiveCache> caches_;
 };
 
 class DnsCacheManagerFactoryImpl : public DnsCacheManagerFactory {
 public:
   DnsCacheManagerFactoryImpl(Singleton::Manager& singleton_manager, Event::Dispatcher& dispatcher,
-                             ThreadLocal::SlotAllocator& tls, Stats::Scope& root_scope)
-      : singleton_manager_(singleton_manager), dispatcher_(dispatcher), tls_(tls),
+                             ThreadLocal::SlotAllocator& tls, Runtime::RandomGenerator& random,
+                             Stats::Scope& root_scope)
+      : singleton_manager_(singleton_manager), dispatcher_(dispatcher), tls_(tls), random_(random),
         root_scope_(root_scope) {}
 
   DnsCacheManagerSharedPtr get() override {
-    return getCacheManager(singleton_manager_, dispatcher_, tls_, root_scope_);
+    return getCacheManager(singleton_manager_, dispatcher_, tls_, random_, root_scope_);
   }
 
 private:
   Singleton::Manager& singleton_manager_;
   Event::Dispatcher& dispatcher_;
   ThreadLocal::SlotAllocator& tls_;
+  Runtime::RandomGenerator& random_;
   Stats::Scope& root_scope_;
 };
 

source/extensions/filters/http/dynamic_forward_proxy/config.cc

@@ -15,7 +15,8 @@ Http::FilterFactoryCb DynamicForwardProxyFilterFactory::createFilterFactoryFromP
     const envoy::extensions::filters::http::dynamic_forward_proxy::v3::FilterConfig& proto_config,
     const std::string&, Server::Configuration::FactoryContext& context) {
   Extensions::Common::DynamicForwardProxy::DnsCacheManagerFactoryImpl cache_manager_factory(
-      context.singletonManager(), context.dispatcher(), context.threadLocal(), context.scope());
+      context.singletonManager(), context.dispatcher(), context.threadLocal(), context.random(),
+      context.scope());
   ProxyFilterConfigSharedPtr filter_config(std::make_shared<ProxyFilterConfig>(
       proto_config, cache_manager_factory, context.clusterManager()));
   return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void {