diff --git a/api/envoy/config/cluster/v3/cluster.proto b/api/envoy/config/cluster/v3/cluster.proto index bd441dea49a0..3e3af8a71412 100644 --- a/api/envoy/config/cluster/v3/cluster.proto +++ b/api/envoy/config/cluster/v3/cluster.proto @@ -109,7 +109,6 @@ message Cluster { // this option or not. CLUSTER_PROVIDED = 6; - // [#not-implemented-hide:] Use the new :ref:`load_balancing_policy // ` field to determine the LB policy. // [#next-major-version: In the v3 API, we should consider deprecating the lb_policy field // and instead using the new load_balancing_policy field as the one and only mechanism for @@ -718,8 +717,7 @@ message Cluster { // The :ref:`load balancer type ` to use // when picking a host in the cluster. - // [#comment:TODO: Remove enum constraint :ref:`LOAD_BALANCING_POLICY_CONFIG` when implemented.] - LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true not_in: 7}]; + LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true}]; // Setting this is required for specifying members of // :ref:`STATIC`, diff --git a/api/envoy/config/cluster/v4alpha/cluster.proto b/api/envoy/config/cluster/v4alpha/cluster.proto index 927bbddaba5e..08cb69ed3ecb 100644 --- a/api/envoy/config/cluster/v4alpha/cluster.proto +++ b/api/envoy/config/cluster/v4alpha/cluster.proto @@ -109,7 +109,6 @@ message Cluster { // this option or not. CLUSTER_PROVIDED = 6; - // [#not-implemented-hide:] Use the new :ref:`load_balancing_policy // ` field to determine the LB policy. // [#next-major-version: In the v3 API, we should consider deprecating the lb_policy field // and instead using the new load_balancing_policy field as the one and only mechanism for @@ -727,8 +726,7 @@ message Cluster { // The :ref:`load balancer type ` to use // when picking a host in the cluster. - // [#comment:TODO: Remove enum constraint :ref:`LOAD_BALANCING_POLICY_CONFIG` when implemented.] - LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true not_in: 7}]; + LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true}]; // Setting this is required for specifying members of // :ref:`STATIC`, diff --git a/generated_api_shadow/envoy/config/cluster/v3/cluster.proto b/generated_api_shadow/envoy/config/cluster/v3/cluster.proto index 7044c50f3b83..06030758708e 100644 --- a/generated_api_shadow/envoy/config/cluster/v3/cluster.proto +++ b/generated_api_shadow/envoy/config/cluster/v3/cluster.proto @@ -106,7 +106,6 @@ message Cluster { // this option or not. CLUSTER_PROVIDED = 6; - // [#not-implemented-hide:] Use the new :ref:`load_balancing_policy // ` field to determine the LB policy. // [#next-major-version: In the v3 API, we should consider deprecating the lb_policy field // and instead using the new load_balancing_policy field as the one and only mechanism for @@ -719,8 +718,7 @@ message Cluster { // The :ref:`load balancer type ` to use // when picking a host in the cluster. - // [#comment:TODO: Remove enum constraint :ref:`LOAD_BALANCING_POLICY_CONFIG` when implemented.] - LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true not_in: 7}]; + LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true}]; // Setting this is required for specifying members of // :ref:`STATIC`, diff --git a/generated_api_shadow/envoy/config/cluster/v4alpha/cluster.proto b/generated_api_shadow/envoy/config/cluster/v4alpha/cluster.proto index a97ade452560..1858a41b05c5 100644 --- a/generated_api_shadow/envoy/config/cluster/v4alpha/cluster.proto +++ b/generated_api_shadow/envoy/config/cluster/v4alpha/cluster.proto @@ -111,7 +111,6 @@ message Cluster { // this option or not. CLUSTER_PROVIDED = 6; - // [#not-implemented-hide:] Use the new :ref:`load_balancing_policy // ` field to determine the LB policy. // [#next-major-version: In the v3 API, we should consider deprecating the lb_policy field // and instead using the new load_balancing_policy field as the one and only mechanism for @@ -727,8 +726,7 @@ message Cluster { // The :ref:`load balancer type ` to use // when picking a host in the cluster. - // [#comment:TODO: Remove enum constraint :ref:`LOAD_BALANCING_POLICY_CONFIG` when implemented.] - LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true not_in: 7}]; + LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true}]; // Setting this is required for specifying members of // :ref:`STATIC`, diff --git a/include/envoy/upstream/load_balancer.h b/include/envoy/upstream/load_balancer.h index fe8bb7e73b92..611ae937bdf7 100644 --- a/include/envoy/upstream/load_balancer.h +++ b/include/envoy/upstream/load_balancer.h @@ -111,6 +111,47 @@ class LoadBalancer { using LoadBalancerPtr = std::unique_ptr; +/** + * Context passed to load balancer factory to access server resources. + */ +class LoadBalancerFactoryContext { +public: + virtual ~LoadBalancerFactoryContext() = default; + + /** + * @return ProtobufMessage::ValidationVisitor& validation visitor for filter configuration + * messages. + */ + virtual ProtobufMessage::ValidationVisitor& messageValidationVisitor() PURE; +}; + +class TypedLoadBalancerFactory : public Config::UntypedFactory { +public: + ~TypedLoadBalancerFactory() override = default; + + virtual LoadBalancerPtr + create(const envoy::config::cluster::v3::LoadBalancingPolicy::Policy& policy, + LoadBalancerType load_balancer_type, LoadBalancerFactoryContext& context, + const PrioritySet& priority_set, const PrioritySet* local_priority_set, + ClusterStats& cluster_stats, Runtime::Loader& loader, Random::RandomGenerator& random, + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) PURE; + + std::string category() const override { return "envoy.load_balancers"; } +}; + +using TypedLoadBalancerFactorySharedPtr = std::shared_ptr; + +class TypedLoadBalancer { +public: + virtual ~TypedLoadBalancer() = default; + + virtual TypedLoadBalancerFactory* factory() PURE; + + virtual void initialize() PURE; +}; + +using TypedLoadBalancerPtr = std::unique_ptr; + /** * Factory for load balancers. */ diff --git a/include/envoy/upstream/load_balancer_type.h b/include/envoy/upstream/load_balancer_type.h index 44d8f708e814..86afde7a6205 100644 --- a/include/envoy/upstream/load_balancer_type.h +++ b/include/envoy/upstream/load_balancer_type.h @@ -22,7 +22,8 @@ enum class LoadBalancerType { RingHash, OriginalDst, Maglev, - ClusterProvided + ClusterProvided, + LoadBalancingPolicyConfig }; /** diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 6a5cb1a151c2..2a95fa65008d 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -787,6 +787,12 @@ class ClusterInfo { return std::dynamic_pointer_cast(extensionProtocolOptions(name)); } + /** + * @return const envoy::config::cluster::v3::Cluster::CommonLbConfig& the common configuration for + * all load balancers for this cluster. + */ + virtual const envoy::config::cluster::v3::LoadBalancingPolicy& loadBalancingPolicy() const PURE; + /** * @return const envoy::config::cluster::v3::Cluster::CommonLbConfig& the common configuration for * all load balancers for this cluster. diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 850e6c5eaadf..805ea11848ca 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -202,6 +202,8 @@ envoy_cc_library( "//include/envoy/upstream:load_balancer_interface", "//include/envoy/upstream:upstream_interface", "//source/common/common:assert_lib", + "//source/common/config:utility_lib", + "//source/common/config:well_known_names", "//source/common/protobuf:utility_lib", "//source/common/runtime:runtime_protos_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index a2f97ff9afc1..944c0a8a30b0 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -253,7 +253,7 @@ ClusterManagerImpl::ClusterManagerImpl( ProtobufMessage::ValidationContext& validation_context, Api::Api& api, Http::Context& http_context, Grpc::Context& grpc_context, Router::Context& router_context) : factory_(factory), runtime_(runtime), stats_(stats), tls_(tls), - random_(api.randomGenerator()), + random_(api.randomGenerator()), validation_context_(validation_context), bind_config_(bootstrap.cluster_manager().upstream_bind_config()), local_info_(local_info), cm_stats_(generateStats(stats)), init_helper_(*this, [this](ClusterManagerCluster& cluster) { onClusterInit(cluster); }), @@ -814,6 +814,9 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust } } else if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided) { cluster_entry_it->second->thread_aware_lb_ = std::move(new_cluster_pair.second); + } else if (cluster_reference.info()->lbType() == LoadBalancerType::LoadBalancingPolicyConfig) { + cluster_entry_it->second->typed_lb_ = std::make_unique( + cluster_reference.info()->loadBalancingPolicy(), cluster_reference.info()->lbConfig()); } updateClusterCounts(); @@ -945,8 +948,10 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_ } LoadBalancerFactorySharedPtr load_balancer_factory; + TypedLoadBalancerFactory* typed_load_balancer_factory; if (add_or_update_cluster) { load_balancer_factory = cm_cluster.loadBalancerFactory(); + typed_load_balancer_factory = cm_cluster.typedLoadBalancerFactory(); } for (auto& per_priority : params.per_priority_update_params_) { @@ -959,7 +964,7 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_ tls_.runOnAllThreads( [info = cm_cluster.cluster().info(), params = std::move(params), add_or_update_cluster, - load_balancer_factory](OptRef cluster_manager) { + load_balancer_factory, typed_load_balancer_factory](OptRef cluster_manager) { ThreadLocalClusterManagerImpl::ClusterEntry* new_cluster = nullptr; if (add_or_update_cluster) { if (cluster_manager->thread_local_clusters_.count(info->name()) > 0) { @@ -969,7 +974,8 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(ClusterManagerCluster& cm_ } new_cluster = new ThreadLocalClusterManagerImpl::ClusterEntry(*cluster_manager, info, - load_balancer_factory); + load_balancer_factory, + typed_load_balancer_factory); cluster_manager->thread_local_clusters_[info->name()].reset(new_cluster); } @@ -1067,7 +1073,8 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ThreadLocalClusterManagerImpl const auto& local_cluster_name = local_cluster_params->info_->name(); ENVOY_LOG(debug, "adding TLS local cluster {}", local_cluster_name); thread_local_clusters_[local_cluster_name] = std::make_unique( - *this, local_cluster_params->info_, local_cluster_params->load_balancer_factory_); + *this, local_cluster_params->info_, local_cluster_params->load_balancer_factory_, + local_cluster_params->typed_load_balancer_factory_); local_priority_set_ = &thread_local_clusters_[local_cluster_name]->priority_set_; } } @@ -1308,8 +1315,9 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::getHttpConnPoolsContainer( ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::ClusterEntry( ThreadLocalClusterManagerImpl& parent, ClusterInfoConstSharedPtr cluster, - const LoadBalancerFactorySharedPtr& lb_factory) - : parent_(parent), lb_factory_(lb_factory), cluster_info_(cluster), + const LoadBalancerFactorySharedPtr& lb_factory, + TypedLoadBalancerFactory* typed_lb_factory) + : parent_(parent), lb_factory_(lb_factory), typed_lb_factory_(typed_lb_factory), cluster_info_(cluster), http_async_client_(cluster, parent.parent_.stats_, parent.thread_local_dispatcher_, parent.parent_.local_info_, parent.parent_, parent.parent_.runtime_, parent.parent_.random_, @@ -1348,6 +1356,23 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::ClusterEntry( parent.parent_.random_, cluster->lbConfig()); break; } + case LoadBalancerType::LoadBalancingPolicyConfig: { + ASSERT(typed_lb_factory != nullptr); + for (const auto& policy : cluster->loadBalancingPolicy().policies()) { + LoadBalancerFactoryContextImpl context( + parent_.parent_.validation_context_.staticValidationVisitor()); + + lb_ = typed_lb_factory_->create(policy, cluster->lbType(), context, priority_set_, + parent_.local_priority_set_, cluster->stats(), + parent.parent_.runtime_, parent.parent_.random_, cluster->lbConfig()); + break; + } + if (lb_ == nullptr) { + ENVOY_LOG(critical, "Didn't find any registered implementation for load_balancing_policy."); + ASSERT(lb_ != nullptr); + } + break; + } case LoadBalancerType::ClusterProvided: case LoadBalancerType::RingHash: case LoadBalancerType::Maglev: diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index ec75f0af9467..aef258220053 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -120,6 +120,8 @@ class ClusterManagerCluster { // Return a new load balancer factory if the cluster has one. virtual LoadBalancerFactorySharedPtr loadBalancerFactory() PURE; + virtual TypedLoadBalancerFactory* typedLoadBalancerFactory() PURE; + // Return true if a cluster has already been added or updated. virtual bool addedOrUpdated() PURE; @@ -390,7 +392,8 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggablefactory(); + } else { + return nullptr; + } + } + bool addedOrUpdated() override { return added_or_updated_; } void setAddedOrUpdated() override { ASSERT(!added_or_updated_); @@ -499,6 +513,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable +class ConfigurableTypedLoadBalancerFactory : public TypedLoadBalancerFactory { + virtual ProtobufTypes::MessagePtr createEmptyConfigProto() { + return std::make_unique(); + } + + LoadBalancerPtr + create(const envoy::config::cluster::v3::LoadBalancingPolicy::Policy& policy, + LoadBalancerType load_balancer_type, LoadBalancerFactoryContext& context, + const PrioritySet& priority_set, const PrioritySet* local_priority_set, + ClusterStats& cluster_stats, Runtime::Loader& loader, Random::RandomGenerator& random, + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) override { + ProtobufTypes::MessagePtr config = createEmptyConfigProto(); + + Envoy::Config::Utility::translateOpaqueConfig(policy.typed_config(), + ProtobufWkt::Struct::default_instance(), + context.messageValidationVisitor(), *config); + + return createLoadBalancerWithConfig(load_balancer_type, priority_set, local_priority_set, + cluster_stats, loader, random, common_config, + MessageUtil::downcastAndValidate( + *config, context.messageValidationVisitor())); + } + + virtual LoadBalancerPtr + createLoadBalancerWithConfig(LoadBalancerType, const PrioritySet&, const PrioritySet*, + ClusterStats&, Runtime::Loader&, Random::RandomGenerator&, + const envoy::config::cluster::v3::Cluster::CommonLbConfig&, + const ConfigProto&) PURE; +}; + +class LoadBalancerFactoryContextImpl : public LoadBalancerFactoryContext { + +public: + LoadBalancerFactoryContextImpl(ProtobufMessage::ValidationVisitor& validation_visitor) + : validation_visitor_(validation_visitor) {} + + ProtobufMessage::ValidationVisitor& messageValidationVisitor() override { + return validation_visitor_; + } + +private: + ProtobufMessage::ValidationVisitor& validation_visitor_; +}; + +class TypedLoadBalancerImpl : public TypedLoadBalancer { +public: + TypedLoadBalancerImpl( + const envoy::config::cluster::v3::LoadBalancingPolicy& lb_policy, + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) + : lb_policy_(lb_policy), common_config_(common_config) { + for (const auto& policy : lb_policy_.policies()) { + factory_ = Registry::FactoryRegistry::getFactory(policy.name()); + if (factory_ == nullptr) { + continue; + } + break; + } + } + + TypedLoadBalancerFactory* factory() { + return factory_; + } + + void initialize() { } + + + const envoy::config::cluster::v3::LoadBalancingPolicy& lb_policy_; + const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config_; + TypedLoadBalancerFactory* factory_; +}; + /** * Base class for all LB implementations. */ diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index 65fc2d241c6a..e6290498dd0d 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -779,6 +779,7 @@ SubsetLoadBalancer::PrioritySubsetImpl::PrioritySubsetImpl(const SubsetLoadBalan lb_ = thread_aware_lb_->factory()->create(); break; + case LoadBalancerType::LoadBalancingPolicyConfig: case LoadBalancerType::OriginalDst: case LoadBalancerType::ClusterProvided: // LoadBalancerType::OriginalDst is blocked in the factory. LoadBalancerType::ClusterProvided diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 83917fb6fc39..9d4407ce5511 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -775,6 +775,7 @@ ClusterInfoImpl::ClusterInfoImpl( added_via_api_(added_via_api), lb_subset_(LoadBalancerSubsetInfoImpl(config.lb_subset_config())), metadata_(config.metadata()), typed_metadata_(config.metadata()), + load_balancing_policy_(config.load_balancing_policy()), common_lb_config_(config.common_lb_config()), cluster_socket_options_(parseClusterSocketOptions(config, bind_config)), drain_connections_on_host_removal_(config.ignore_health_on_host_removal()), @@ -830,6 +831,9 @@ ClusterInfoImpl::ClusterInfoImpl( lb_type_ = LoadBalancerType::ClusterProvided; break; + case envoy::config::cluster::v3::Cluster::LOAD_BALANCING_POLICY_CONFIG: + lb_type_ = LoadBalancerType::LoadBalancingPolicyConfig; + break; default: NOT_REACHED_GCOVR_EXCL_LINE; } diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index eb38961922c3..f6e221ebfe4a 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -542,6 +542,9 @@ class ClusterInfoImpl : public ClusterInfo, // Upstream::ClusterInfo bool addedViaApi() const override { return added_via_api_; } + const envoy::config::cluster::v3::LoadBalancingPolicy& loadBalancingPolicy() const override { + return load_balancing_policy_; + } const envoy::config::cluster::v3::Cluster::CommonLbConfig& lbConfig() const override { return common_lb_config_; } @@ -719,6 +722,7 @@ class ClusterInfoImpl : public ClusterInfo, LoadBalancerSubsetInfoImpl lb_subset_; const envoy::config::core::v3::Metadata metadata_; Envoy::Config::TypedMetadataImpl typed_metadata_; + const envoy::config::cluster::v3::LoadBalancingPolicy load_balancing_policy_; const envoy::config::cluster::v3::Cluster::CommonLbConfig common_lb_config_; const Network::ConnectionSocket::OptionsSharedPtr cluster_socket_options_; const bool drain_connections_on_host_removal_; diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc index 063dfc76f6cd..e1c6c8763ab5 100644 --- a/test/mocks/upstream/cluster_info.cc +++ b/test/mocks/upstream/cluster_info.cc @@ -102,6 +102,7 @@ MockClusterInfo::MockClusterInfo() ON_CALL(*this, lbMaglevConfig()).WillByDefault(ReturnRef(lb_maglev_config_)); ON_CALL(*this, lbOriginalDstConfig()).WillByDefault(ReturnRef(lb_original_dst_config_)); ON_CALL(*this, upstreamConfig()).WillByDefault(ReturnRef(upstream_config_)); + ON_CALL(*this, loadBalancingPolicy()).WillByDefault(ReturnRef(load_balancing_policy_)); ON_CALL(*this, lbConfig()).WillByDefault(ReturnRef(lb_config_)); ON_CALL(*this, clusterSocketOptions()).WillByDefault(ReturnRef(cluster_socket_options_)); ON_CALL(*this, metadata()).WillByDefault(ReturnRef(metadata_)); diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index 31a941bb53ff..760af33e638f 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -104,6 +104,8 @@ class MockClusterInfo : public ClusterInfo { (const)); MOCK_METHOD(ProtocolOptionsConfigConstSharedPtr, extensionProtocolOptions, (const std::string&), (const)); + MOCK_METHOD(const envoy::config::cluster::v3::LoadBalancingPolicy&, loadBalancingPolicy, (), + (const)); MOCK_METHOD(const envoy::config::cluster::v3::Cluster::CommonLbConfig&, lbConfig, (), (const)); MOCK_METHOD(LoadBalancerType, lbType, (), (const)); MOCK_METHOD(envoy::config::cluster::v3::Cluster::DiscoveryType, type, (), (const)); @@ -190,6 +192,7 @@ class MockClusterInfo : public ClusterInfo { absl::optional lb_original_dst_config_; absl::optional upstream_config_; Network::ConnectionSocket::OptionsSharedPtr cluster_socket_options_; + envoy::config::cluster::v3::LoadBalancingPolicy load_balancing_policy_; envoy::config::cluster::v3::Cluster::CommonLbConfig lb_config_; envoy::config::core::v3::Metadata metadata_; std::unique_ptr typed_metadata_;