From 8468cf26bf2facd576c549fd90cdb066f8204a10 Mon Sep 17 00:00:00 2001 From: Adam Anderson <6754028+AdamEAnderson@users.noreply.github.com> Date: Wed, 9 Aug 2023 20:45:29 -0700 Subject: [PATCH] Enforce locality order in upstream HostsPerLocality (#28527) * enforce locality order Signed-off-by: Adam Anderson Signed-off-by: Adam Anderson <6754028+AdamEAnderson@users.noreply.github.com> * add invariant to description of HostsPerLocality::get() Signed-off-by: Adam Anderson Signed-off-by: Adam Anderson <6754028+AdamEAnderson@users.noreply.github.com> * remove invariant assertion, add PriorityStateManager test instead Signed-off-by: Adam Anderson <6754028+AdamEAnderson@users.noreply.github.com> * fix tests Signed-off-by: Adam Anderson <6754028+AdamEAnderson@users.noreply.github.com> --------- Signed-off-by: Adam Anderson <6754028+AdamEAnderson@users.noreply.github.com> --- envoy/upstream/upstream.h | 5 +- source/common/upstream/upstream_impl.h | 14 + test/common/upstream/BUILD | 1 + test/common/upstream/hds_test.cc | 8 +- .../upstream/load_balancer_impl_test.cc | 221 +++++++++----- test/common/upstream/subset_lb_test.cc | 81 ++++-- test/common/upstream/upstream_impl_test.cc | 269 ++++++++++++++---- test/common/upstream/utility.h | 22 ++ test/extensions/clusters/eds/eds_test.cc | 49 ++++ .../maglev/maglev_lb_test.cc | 39 ++- .../ring_hash/ring_hash_lb_test.cc | 77 +++-- test/server/admin/config_dump_handler_test.cc | 88 +++--- 12 files changed, 652 insertions(+), 222 deletions(-) diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h index a57a66cb8b4e..f350af9b7bae 100644 --- a/envoy/upstream/upstream.h +++ b/envoy/upstream/upstream.h @@ -321,7 +321,10 @@ class HostsPerLocality { /** * @return const std::vector& list of hosts organized per * locality. The local locality is the first entry if - * hasLocalLocality() is true. + * hasLocalLocality() is true. All hosts within the same entry have the same locality + * and all hosts with a given locality are in the same entry. With the exception of + * the local locality entry (if present), all entries are sorted by locality with + * those considered less by the LocalityLess comparator ordered earlier in the list. */ virtual const std::vector& get() const PURE; diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 05a097d349db..349f01cbc79a 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -495,9 +495,23 @@ class HostsPerLocalityImpl : public HostsPerLocality { HostsPerLocalityImpl() : HostsPerLocalityImpl(std::vector(), false) {} // Single locality constructor + // + // Parameter requirements: + // 1. All entries in hosts must have the same locality. + // 2. If has_local_locality is true, then the locality of all entries in hosts + // must be equal to the current envoy's locality. HostsPerLocalityImpl(const HostVector& hosts, bool has_local_locality = false) : HostsPerLocalityImpl(std::vector({hosts}), has_local_locality) {} + // Multiple localities constructor + // + // locality_hosts must adhere to the following ordering constraints: + // 1. All hosts within a single HostVector bucket must have the same locality + // 2. No hosts in different HostVector buckets can have the same locality + // 3. If has_local_locality is true, then the locality of all hosts in the first HostVector bucket + // must be equal to the current envoy's locality. + // 4. All non-local HostVector buckets must be sorted in ascending order by the LocalityLess + // comparator HostsPerLocalityImpl(std::vector&& locality_hosts, bool has_local_locality) : local_(has_local_locality), hosts_per_locality_(std::move(locality_hosts)) { ASSERT(!has_local_locality || !hosts_per_locality_.empty()); diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 4b5bf65ce077..af3bfca7260a 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -276,6 +276,7 @@ envoy_cc_test( "//test/test_common:simulated_time_system_lib", "//test/test_common:test_runtime_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", ], ) diff --git a/test/common/upstream/hds_test.cc b/test/common/upstream/hds_test.cc index dd51ad0a6152..e6430bdc9cba 100644 --- a/test/common/upstream/hds_test.cc +++ b/test/common/upstream/hds_test.cc @@ -290,7 +290,9 @@ TEST_F(HdsTest, TestProcessMessageEndpoints) { auto* health_check = message->add_cluster_health_checks(); health_check->set_cluster_name("anna" + std::to_string(i)); for (int j = 0; j < 3; j++) { - auto* address = health_check->add_locality_endpoints()->add_endpoints()->mutable_address(); + auto* locality_endpoints = health_check->add_locality_endpoints(); + locality_endpoints->mutable_locality()->set_zone(std::to_string(j)); + auto* address = locality_endpoints->add_endpoints()->mutable_address(); address->mutable_socket_address()->set_address("127.0.0." + std::to_string(i)); address->mutable_socket_address()->set_port_value(1234 + j); } @@ -1208,7 +1210,9 @@ TEST_F(HdsTest, TestCustomHealthCheckPortWhenCreate) { auto* health_check = message->add_cluster_health_checks(); health_check->set_cluster_name("anna"); for (int i = 0; i < 3; i++) { - auto* endpoint = health_check->add_locality_endpoints()->add_endpoints(); + auto* locality_endpoints = health_check->add_locality_endpoints(); + locality_endpoints->mutable_locality()->set_zone(std::to_string(i)); + auto* endpoint = locality_endpoints->add_endpoints(); endpoint->mutable_health_check_config()->set_port_value(4321 + i); auto* address = endpoint->mutable_address(); address->mutable_socket_address()->set_address("127.0.0.1"); diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index c620fa879e50..8bf405ca46a9 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -7,6 +7,8 @@ #include #include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/core/v3/health_check.pb.h" #include "source/common/network/utility.h" #include "source/common/upstream/load_balancer_impl.h" @@ -989,9 +991,17 @@ TEST_P(RoundRobinLoadBalancerTest, Seed) { } TEST_P(RoundRobinLoadBalancerTest, Locality) { - HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:82", simTime())})); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + + HostVectorSharedPtr hosts( + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)})); HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{(*hosts)[1]}, {(*hosts)[0]}, {(*hosts)[2]}}); hostSet().hosts_ = *hosts; @@ -1019,9 +1029,15 @@ TEST_P(RoundRobinLoadBalancerTest, Locality) { } TEST_P(RoundRobinLoadBalancerTest, DegradedLocality) { - HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:84", simTime())})); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + + HostVectorSharedPtr hosts( + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:84", simTime(), zone_b)})); HostVectorSharedPtr healthy_hosts(new HostVector({(*hosts)[0]})); HostVectorSharedPtr degraded_hosts(new HostVector({(*hosts)[1], (*hosts)[2]})); HostsPerLocalitySharedPtr hosts_per_locality = @@ -1199,11 +1215,17 @@ TEST_P(RoundRobinLoadBalancerTest, DisablePanicMode) { TEST_P(RoundRobinLoadBalancerTest, HostSelectionWithFilter) { NiceMock context; - HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime())})); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + + HostVectorSharedPtr hosts( + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)})); HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:80", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:81", simTime())}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)}}); hostSet().hosts_ = *hosts; hostSet().healthy_hosts_ = *hosts; @@ -1242,13 +1264,21 @@ TEST_P(RoundRobinLoadBalancerTest, HostSelectionWithFilter) { } TEST_P(RoundRobinLoadBalancerTest, ZoneAwareSmallCluster) { - HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:82", simTime())})); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + + HostVectorSharedPtr hosts( + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)})); HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:81", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:80", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:82", simTime())}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)}, + {makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)}}); hostSet().hosts_ = *hosts; hostSet().healthy_hosts_ = *hosts; @@ -1288,16 +1318,24 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareDifferentZoneSize) { if (&hostSet() == &failover_host_set_) { // P = 1 does not support zone-aware routing. return; } - HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:82", simTime())})); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + + HostVectorSharedPtr hosts( + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)})); HostsPerLocalitySharedPtr upstream_hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:81", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:80", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:82", simTime())}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_b)}, + {makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)}}); HostsPerLocalitySharedPtr local_hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:81", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:80", simTime())}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:0", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:1", simTime(), zone_b)}}); hostSet().healthy_hosts_ = *hosts; hostSet().hosts_ = *hosts; @@ -1323,13 +1361,20 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareRoutingLargeZoneSwitchOnOff) { if (&hostSet() == &failover_host_set_) { // P = 1 does not support zone-aware routing. return; } - HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:82", simTime())})); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + HostVectorSharedPtr hosts( + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)})); HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:81", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:80", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:82", simTime())}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)}, + {makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)}}); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) .WillRepeatedly(Return(50)); @@ -1360,28 +1405,34 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareRoutingSmallZone) { if (&hostSet() == &failover_host_set_) { // P = 1 does not support zone-aware routing. return; } + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); HostVectorSharedPtr upstream_hosts( - new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:82", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:83", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:84", simTime())})); + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:83", simTime(), zone_c), + makeTestHost(info_, "tcp://127.0.0.1:84", simTime(), zone_c)})); HostVectorSharedPtr local_hosts( - new HostVector({makeTestHost(info_, "tcp://127.0.0.1:0", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:1", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:2", simTime())})); + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:0", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:1", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:2", simTime(), zone_c)})); HostsPerLocalitySharedPtr upstream_hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:81", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:82", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:83", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:84", simTime())}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_b)}, + {makeTestHost(info_, "tcp://127.0.0.1:83", simTime(), zone_c), + makeTestHost(info_, "tcp://127.0.0.1:84", simTime(), zone_c)}}); HostsPerLocalitySharedPtr local_hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:0", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:1", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:2", simTime())}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:0", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:1", simTime(), zone_b)}, + {makeTestHost(info_, "tcp://127.0.0.1:2", simTime(), zone_c)}}); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) .WillRepeatedly(Return(50)); @@ -1411,6 +1462,10 @@ TEST_P(RoundRobinLoadBalancerTest, LowPrecisionForDistribution) { if (&hostSet() == &failover_host_set_) { // P = 1 does not support zone-aware routing. return; } + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); // upstream_hosts and local_hosts do not matter, zone aware routing is based on per zone hosts. HostVectorSharedPtr upstream_hosts( new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime())})); @@ -1431,31 +1486,32 @@ TEST_P(RoundRobinLoadBalancerTest, LowPrecisionForDistribution) { // The following host distribution with current precision should lead to the no_capacity_left // situation. - // Reuse the same host in all of the structures below to reduce time test takes and this does - // not impact load balancing logic. - HostSharedPtr host = makeTestHost(info_, "tcp://127.0.0.1:80", simTime()); + // Reuse the same host for each zone in all of the structures below to reduce time test takes and + // this does not impact load balancing logic. + HostSharedPtr host_a = makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a); + HostSharedPtr host_b = makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_b); HostVector current(45000); for (int i = 0; i < 45000; ++i) { - current[i] = host; + current[i] = host_a; } local_hosts_per_locality.push_back(current); current.resize(55000); for (int i = 0; i < 55000; ++i) { - current[i] = host; + current[i] = host_b; } local_hosts_per_locality.push_back(current); current.resize(44999); for (int i = 0; i < 44999; ++i) { - current[i] = host; + current[i] = host_a; } upstream_hosts_per_locality.push_back(current); current.resize(55001); for (int i = 0; i < 55001; ++i) { - current[i] = host; + current[i] = host_b; } upstream_hosts_per_locality.push_back(current); @@ -1490,12 +1546,17 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingOneZone) { } TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNotHealthy) { - HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.2:80", simTime())})); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostVectorSharedPtr hosts( + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.2:80", simTime(), zone_a)})); HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{}, - {makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.2:80", simTime())}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.2:80", simTime(), zone_a)}}, + true); hostSet().healthy_hosts_ = *hosts; hostSet().hosts_ = *hosts; @@ -1512,15 +1573,19 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingLocalEmpty) { if (&hostSet() == &failover_host_set_) { // P = 1 does not support zone-aware routing. return; } + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); HostVectorSharedPtr upstream_hosts( - new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime())})); + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)})); HostVectorSharedPtr local_hosts(new HostVector({}, {})); HostsPerLocalitySharedPtr upstream_hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:80", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:81", simTime())}}); - HostsPerLocalitySharedPtr local_hosts_per_locality = makeHostsPerLocality({{}, {}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)}}); + HostsPerLocalitySharedPtr local_hosts_per_locality = makeHostsPerLocality({{{}}, {{}}}); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) .WillOnce(Return(50)) @@ -1548,15 +1613,21 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingLocalEmptyFailTrafficOnPani if (&hostSet() == &failover_host_set_) { // P = 1 does not support zone-aware routing. return; } + + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostVectorSharedPtr upstream_hosts( - new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime())})); + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)})); HostVectorSharedPtr local_hosts(new HostVector({}, {})); HostsPerLocalitySharedPtr upstream_hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:80", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:81", simTime())}}); - HostsPerLocalitySharedPtr local_hosts_per_locality = makeHostsPerLocality({{}, {}}); + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)}}); + HostsPerLocalitySharedPtr local_hosts_per_locality = makeHostsPerLocality({{{}}, {{}}}); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 50)) .WillOnce(Return(50)) @@ -1585,14 +1656,18 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNoLocalLocality) { if (&hostSet() == &failover_host_set_) { // P = 1 does not support zone-aware routing. return; } + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); HostVectorSharedPtr upstream_hosts( - new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime())})); - HostVectorSharedPtr local_hosts(new HostVector({}, {})); + new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)})); + HostVectorSharedPtr local_hosts(new HostVector()); HostsPerLocalitySharedPtr upstream_hosts_per_locality = - makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:80", simTime())}, - {makeTestHost(info_, "tcp://127.0.0.1:81", simTime())}}, + makeHostsPerLocality({{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a)}, + {makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)}}, true); const HostsPerLocalitySharedPtr& local_hosts_per_locality = upstream_hosts_per_locality; diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index 56cd3ccc3d73..2c2d80bd1c9a 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -221,23 +221,28 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, void configureWeightedHostSet(const HostURLMetadataMap& first_locality_host_metadata, const HostURLMetadataMap& second_locality_host_metadata, MockHostSet& host_set, LocalityWeights locality_weights) { - HostVector first_locality; HostVector all_hosts; + HostVector first_locality_hosts; + envoy::config::core::v3::Locality first_locality; + first_locality.set_zone("0"); for (const auto& it : first_locality_host_metadata) { - auto host = makeHost(it.first, it.second); - first_locality.emplace_back(host); + auto host = makeHost(it.first, it.second, first_locality); + first_locality_hosts.emplace_back(host); all_hosts.emplace_back(host); } - HostVector second_locality; + envoy::config::core::v3::Locality second_locality; + second_locality.set_zone("1"); + HostVector second_locality_hosts; for (const auto& it : second_locality_host_metadata) { - auto host = makeHost(it.first, it.second); - second_locality.emplace_back(host); + auto host = makeHost(it.first, it.second, second_locality); + second_locality_hosts.emplace_back(host); all_hosts.emplace_back(host); } host_set.hosts_ = all_hosts; - host_set.hosts_per_locality_ = makeHostsPerLocality({first_locality, second_locality}); + host_set.hosts_per_locality_ = + makeHostsPerLocality({first_locality_hosts, second_locality_hosts}); host_set.healthy_hosts_ = host_set.hosts_; host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; host_set.locality_weights_ = std::make_shared(locality_weights); @@ -286,12 +291,22 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, const std::vector& local_host_metadata_per_locality) { EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); + std::vector> localities; + for (uint32_t i = 0; i < 10; ++i) { + envoy::config::core::v3::Locality locality; + locality.set_zone(std::to_string(i)); + localities.emplace_back(std::make_shared(locality)); + } + ASSERT(host_metadata_per_locality.size() <= localities.size()); + ASSERT(local_host_metadata_per_locality.size() <= localities.size()); + HostVector hosts; std::vector hosts_per_locality; - for (const auto& host_metadata : host_metadata_per_locality) { + for (uint32_t i = 0; i < host_metadata_per_locality.size(); ++i) { + const auto& host_metadata = host_metadata_per_locality[i]; HostVector locality_hosts; for (const auto& host_entry : host_metadata) { - HostSharedPtr host = makeHost(host_entry.first, host_entry.second); + HostSharedPtr host = makeHost(host_entry.first, host_entry.second, *localities[i]); hosts.emplace_back(host); locality_hosts.emplace_back(host); } @@ -306,10 +321,11 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, local_hosts_ = std::make_shared(); std::vector local_hosts_per_locality_vector; - for (const auto& local_host_metadata : local_host_metadata_per_locality) { + for (uint32_t i = 0; i < local_host_metadata_per_locality.size(); ++i) { + const auto& local_host_metadata = local_host_metadata_per_locality[i]; HostVector local_locality_hosts; for (const auto& host_entry : local_host_metadata) { - HostSharedPtr host = makeHost(host_entry.first, host_entry.second); + HostSharedPtr host = makeHost(host_entry.first, host_entry.second, *localities[i]); local_hosts_->emplace_back(host); local_locality_hosts.emplace_back(host); } @@ -344,6 +360,18 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, return makeTestHost(info_, url, m, simTime()); } + + HostSharedPtr makeHost(const std::string& url, const HostMetadata& metadata, + const envoy::config::core::v3::Locality& locality) { + envoy::config::core::v3::Metadata m; + for (const auto& m_it : metadata) { + Config::Metadata::mutableMetadataValue(m, Config::MetadataFilters::get().ENVOY_LB, m_it.first) + .set_string_value(m_it.second); + } + + return makeTestHost(info_, url, m, locality, simTime()); + } + HostSharedPtr makeHost(const std::string& url, const HostListMetadata& metadata) { envoy::config::core::v3::Metadata m; for (const auto& m_it : metadata) { @@ -1666,11 +1694,14 @@ TEST_P(SubsetLoadBalancerTest, ZoneAwareFallbackAfterUpdate) { EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][0], lb_->chooseHost(nullptr)); - modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}})}, {host_set_.hosts_[0]}, - absl::optional(0)); + envoy::config::core::v3::Locality local_locality; + local_locality.set_zone("0"); + + modifyHosts({makeHost("tcp://127.0.0.1:8000", {{"version", "1.0"}}, local_locality)}, + {host_set_.hosts_[0]}, absl::optional(0)); - modifyLocalHosts({makeHost("tcp://127.0.0.1:9000", {{"version", "1.0"}})}, {local_hosts_->at(0)}, - 0); + modifyLocalHosts({makeHost("tcp://127.0.0.1:9000", {{"version", "1.0"}}, local_locality)}, + {local_hosts_->at(0)}, 0); EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(nullptr)); @@ -1793,11 +1824,14 @@ TEST_P(SubsetLoadBalancerTest, ZoneAwareFallbackDefaultSubsetAfterUpdate) { EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(nullptr)); - modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "default"}})}, {host_set_.hosts_[1]}, - absl::optional(0)); + envoy::config::core::v3::Locality local_locality; + local_locality.set_zone("0"); + + modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "default"}}, local_locality)}, + {host_set_.hosts_[1]}, absl::optional(0)); modifyLocalHosts({local_hosts_->at(1)}, - {makeHost("tcp://127.0.0.1:9001", {{"version", "default"}})}, 0); + {makeHost("tcp://127.0.0.1:9001", {{"version", "default"}}, local_locality)}, 0); EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(nullptr)); @@ -1916,11 +1950,14 @@ TEST_P(SubsetLoadBalancerTest, ZoneAwareBalancesSubsetsAfterUpdate) { EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[1][1], lb_->chooseHost(&context)); - modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "1.1"}})}, {host_set_.hosts_[1]}, - absl::optional(0)); + envoy::config::core::v3::Locality local_locality; + local_locality.set_zone("0"); + + modifyHosts({makeHost("tcp://127.0.0.1:8001", {{"version", "1.1"}}, local_locality)}, + {host_set_.hosts_[1]}, absl::optional(0)); - modifyLocalHosts({local_hosts_->at(1)}, {makeHost("tcp://127.0.0.1:9001", {{"version", "1.1"}})}, - 0); + modifyLocalHosts({local_hosts_->at(1)}, + {makeHost("tcp://127.0.0.1:9001", {{"version", "1.1"}}, local_locality)}, 0); EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); EXPECT_EQ(host_set_.healthy_hosts_per_locality_->get()[0][1], lb_->chooseHost(&context)); diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 17b9f0bfb5fa..73f82b7e6e5f 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -4873,6 +4873,7 @@ class HostsWithLocalityImpl : public Event::TestUsingSimulatedTime, public testi // Validate HostsPerLocalityImpl constructors. TEST_F(HostsWithLocalityImpl, Cons) { + { const HostsPerLocalityImpl hosts_per_locality; EXPECT_FALSE(hosts_per_locality.hasLocalLocality()); @@ -4880,8 +4881,12 @@ TEST_F(HostsWithLocalityImpl, Cons) { } MockClusterMockPrioritySet cluster; - HostSharedPtr host_0 = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", simTime(), 1); - HostSharedPtr host_1 = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", simTime(), 1); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostSharedPtr host_0 = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", simTime(), zone_a, 1); + HostSharedPtr host_1 = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", simTime(), zone_b, 1); { std::vector locality_hosts = {{host_0}, {host_1}}; @@ -4902,8 +4907,12 @@ TEST_F(HostsWithLocalityImpl, Cons) { TEST_F(HostsWithLocalityImpl, Filter) { MockClusterMockPrioritySet cluster; - HostSharedPtr host_0 = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", simTime(), 1); - HostSharedPtr host_1 = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", simTime(), 1); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostSharedPtr host_0 = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", simTime(), zone_a, 1); + HostSharedPtr host_1 = makeTestHost(cluster.info_, "tcp://10.0.0.1:1234", simTime(), zone_b, 1); { std::vector locality_hosts = {{host_0}, {host_1}}; @@ -4933,12 +4942,6 @@ class HostSetImplLocalityTest : public Event::TestUsingSimulatedTime, public tes LocalityWeightsConstSharedPtr locality_weights_; HostSetImpl host_set_{0, false, kDefaultOverProvisioningFactor}; std::shared_ptr info_{new NiceMock()}; - HostVector hosts_{makeTestHost(info_, "tcp://127.0.0.1:80", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:81", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:82", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:83", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:84", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:85", simTime())}; }; // When no locality weights belong to the host set, there's an empty pick. @@ -4949,32 +4952,52 @@ TEST_F(HostSetImplLocalityTest, Empty) { // When no hosts are healthy we should fail to select a locality TEST_F(HostSetImplLocalityTest, AllUnhealthy) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + HostVector hosts{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)}; + HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}, {hosts_[2]}}); + makeHostsPerLocality({{hosts[0]}, {hosts[1]}, {hosts[2]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1, 1}}; - auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality), locality_weights, {}, {}, - absl::nullopt); + auto hosts_const_shared = std::make_shared(hosts); + host_set_.updateHosts(updateHostsParams(hosts_const_shared, hosts_per_locality), locality_weights, + {}, {}, absl::nullopt); EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); } // When a locality has endpoints that have not yet been warmed, weight calculation should ignore // these hosts. TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { - // We have two localities with 3 hosts in L1, 2 hosts in L2. Two of the hosts in L1 are not + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostVector hosts{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:83", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:84", simTime(), zone_b)}; + + // We have two localities with 3 hosts in A, 2 hosts in B. Two of the hosts in A are not // warmed yet, so even though they are unhealthy we should not adjust the locality weight. HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{hosts_[0], hosts_[1], hosts_[2]}, {hosts_[3], hosts_[4]}}); + makeHostsPerLocality({{hosts[0], hosts[1], hosts[2]}, {hosts[3], hosts[4]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1}}; - auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); + auto hosts_const_shared = std::make_shared(hosts); HostsPerLocalitySharedPtr healthy_hosts_per_locality = - makeHostsPerLocality({{hosts_[0]}, {hosts_[3], hosts_[4]}}); + makeHostsPerLocality({{hosts[0]}, {hosts[3], hosts[4]}}); HostsPerLocalitySharedPtr excluded_hosts_per_locality = - makeHostsPerLocality({{hosts_[1], hosts_[2]}, {}}); + makeHostsPerLocality({{hosts[1], hosts[2]}, {}}); host_set_.updateHosts( HostSetImpl::updateHostsParams( - hosts, hosts_per_locality, + hosts_const_shared, hosts_per_locality, makeHostsFromHostsPerLocality(healthy_hosts_per_locality), healthy_hosts_per_locality, std::make_shared(), HostsPerLocalityImpl::empty(), @@ -4990,12 +5013,18 @@ TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { // When a locality has zero hosts, it should be treated as if it has zero healthy. TEST_F(HostSetImplLocalityTest, EmptyLocality) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + HostVector hosts{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_a)}; + HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{hosts_[0], hosts_[1], hosts_[2]}, {}}); + makeHostsPerLocality({{hosts[0], hosts[1], hosts[2]}, {}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1}}; - auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), + auto hosts_const_shared = std::make_shared(hosts); + host_set_.updateHosts(updateHostsParams(hosts_const_shared, hosts_per_locality, + std::make_shared(hosts), hosts_per_locality), locality_weights, {}, {}, absl::nullopt); // Verify that we are not RRing between localities. @@ -5005,11 +5034,18 @@ TEST_F(HostSetImplLocalityTest, EmptyLocality) { // When all locality weights are zero we should fail to select a locality. TEST_F(HostSetImplLocalityTest, AllZeroWeights) { - HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}}); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostVector hosts{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)}; + + HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{hosts[0]}, {hosts[1]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{0, 0}}; - auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), + auto hosts_const_shared = std::make_shared(hosts); + host_set_.updateHosts(updateHostsParams(hosts_const_shared, hosts_per_locality, + std::make_shared(hosts), hosts_per_locality), locality_weights, {}, {}); EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); @@ -5017,12 +5053,22 @@ TEST_F(HostSetImplLocalityTest, AllZeroWeights) { // When all locality weights are the same we have unweighted RR behavior. TEST_F(HostSetImplLocalityTest, Unweighted) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + HostVector hosts{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)}; + HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}, {hosts_[2]}}); + makeHostsPerLocality({{hosts[0]}, {hosts[1]}, {hosts[2]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1, 1}}; - auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), + auto hosts_const_shared = std::make_shared(hosts); + host_set_.updateHosts(updateHostsParams(hosts_const_shared, hosts_per_locality, + std::make_shared(hosts), hosts_per_locality), locality_weights, {}, {}, absl::nullopt); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); @@ -5035,11 +5081,18 @@ TEST_F(HostSetImplLocalityTest, Unweighted) { // When locality weights differ, we have weighted RR behavior. TEST_F(HostSetImplLocalityTest, Weighted) { - HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}}); + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostVector hosts{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b)}; + + HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{hosts[0]}, {hosts[1]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 2}}; - auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), + auto hosts_const_shared = std::make_shared(hosts); + host_set_.updateHosts(updateHostsParams(hosts_const_shared, hosts_per_locality, + std::make_shared(hosts), hosts_per_locality), locality_weights, {}, {}, absl::nullopt); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); @@ -5052,12 +5105,22 @@ TEST_F(HostSetImplLocalityTest, Weighted) { // Localities with no weight assignment are never picked. TEST_F(HostSetImplLocalityTest, MissingWeight) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + HostVector hosts{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_c)}; + HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}, {hosts_[2]}}); + makeHostsPerLocality({{hosts[0]}, {hosts[1]}, {hosts[2]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 0, 1}}; - auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), + auto hosts_const_shared = std::make_shared(hosts); + host_set_.updateHosts(updateHostsParams(hosts_const_shared, hosts_per_locality, + std::make_shared(hosts), hosts_per_locality), locality_weights, {}, {}, absl::nullopt); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); @@ -5070,16 +5133,27 @@ TEST_F(HostSetImplLocalityTest, MissingWeight) { // Gentle failover between localities as health diminishes. TEST_F(HostSetImplLocalityTest, UnhealthyFailover) { - const auto setHealthyHostCount = [this](uint32_t host_count) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostVector hosts{makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:82", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:83", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:84", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:85", simTime(), zone_b)}; + + const auto setHealthyHostCount = [this, hosts](uint32_t host_count) { LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 2}}; - HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality( - {{hosts_[0], hosts_[1], hosts_[2], hosts_[3], hosts_[4]}, {hosts_[5]}}); + HostsPerLocalitySharedPtr hosts_per_locality = + makeHostsPerLocality({{hosts[0], hosts[1], hosts[2], hosts[3], hosts[4]}, {hosts[5]}}); HostVector healthy_hosts; for (uint32_t i = 0; i < host_count; ++i) { - healthy_hosts.emplace_back(hosts_[i]); + healthy_hosts.emplace_back(hosts[i]); } HostsPerLocalitySharedPtr healthy_hosts_per_locality = - makeHostsPerLocality({healthy_hosts, {hosts_[5]}}); + makeHostsPerLocality({healthy_hosts, {hosts[5]}}); auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, @@ -5121,9 +5195,13 @@ TEST(OverProvisioningFactorTest, LocalityPickChanges) { HostSetImpl host_set(0, false, overprovisioning_factor); std::shared_ptr cluster_info{new NiceMock()}; auto time_source = std::make_unique>(); - HostVector hosts{makeTestHost(cluster_info, "tcp://127.0.0.1:80", *time_source), - makeTestHost(cluster_info, "tcp://127.0.0.1:81", *time_source), - makeTestHost(cluster_info, "tcp://127.0.0.1:82", *time_source)}; + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostVector hosts{makeTestHost(cluster_info, "tcp://127.0.0.1:80", *time_source, zone_a), + makeTestHost(cluster_info, "tcp://127.0.0.1:81", *time_source, zone_a), + makeTestHost(cluster_info, "tcp://127.0.0.1:82", *time_source, zone_b)}; LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1}}; HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{hosts[0], hosts[1]}, {hosts[2]}}); @@ -5163,11 +5241,15 @@ TEST(OverProvisioningFactorTest, LocalityPickChanges) { TEST(HostPartitionTest, PartitionHosts) { std::shared_ptr info{new NiceMock()}; auto time_source = std::make_unique>(); - HostVector hosts{makeTestHost(info, "tcp://127.0.0.1:80", *time_source), - makeTestHost(info, "tcp://127.0.0.1:81", *time_source), - makeTestHost(info, "tcp://127.0.0.1:82", *time_source), - makeTestHost(info, "tcp://127.0.0.1:83", *time_source), - makeTestHost(info, "tcp://127.0.0.1:84", *time_source)}; + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + HostVector hosts{makeTestHost(info, "tcp://127.0.0.1:80", *time_source, zone_a), + makeTestHost(info, "tcp://127.0.0.1:81", *time_source, zone_a), + makeTestHost(info, "tcp://127.0.0.1:82", *time_source, zone_b), + makeTestHost(info, "tcp://127.0.0.1:83", *time_source, zone_b), + makeTestHost(info, "tcp://127.0.0.1:84", *time_source, zone_b)}; hosts[0]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); hosts[1]->healthFlagSet(Host::HealthFlag::DEGRADED_ACTIVE_HC); @@ -5261,6 +5343,87 @@ TEST_F(ClusterInfoImplTest, FilterChain) { cluster->info()->createFilterChain(manager); } +class PriorityStateManagerTest : public Event::TestUsingSimulatedTime, + public testing::Test, + public UpstreamImplTestBase {}; + +TEST_F(PriorityStateManagerTest, LocalityClusterUpdate) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + + server_context_.local_info_.node_.mutable_locality()->CopyFrom(zone_a); + + // Construct cluster to use for updates + const std::string cluster_yaml = R"EOF( + name: staticcluster + connect_timeout: 0.25s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + endpoints: + - locality: + zone: B + lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 80 + )EOF"; + envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(cluster_yaml); + + Envoy::Upstream::ClusterFactoryContextImpl factory_context( + server_context_, server_context_.cluster_manager_, nullptr, ssl_context_manager_, nullptr, + false); + StaticClusterImpl cluster(cluster_config, factory_context); + cluster.initialize([] {}); + EXPECT_EQ(1UL, cluster.prioritySet().hostSetsPerPriority()[0]->hosts().size()); + + // Make priority state manager and fill it with the initial state of the cluster and the added + // hosts + PriorityStateManager priority_state_manager(cluster, server_context_.local_info_, nullptr); + + auto current_hosts = cluster.prioritySet().hostSetsPerPriority()[0]->hosts(); + HostVector hosts_added{makeTestHost(cluster.info(), "tcp://127.0.0.1:81", simTime(), zone_b), + makeTestHost(cluster.info(), "tcp://127.0.0.1:82", simTime(), zone_a)}; + + envoy::config::endpoint::v3::LocalityLbEndpoints zone_a_endpoints; + zone_a_endpoints.mutable_locality()->CopyFrom(zone_a); + envoy::config::endpoint::v3::LocalityLbEndpoints zone_b_endpoints = + cluster_config.load_assignment().endpoints()[0]; + + priority_state_manager.initializePriorityFor(zone_b_endpoints); + priority_state_manager.registerHostForPriority(hosts_added[0], zone_b_endpoints); + priority_state_manager.registerHostForPriority( + cluster.prioritySet().hostSetsPerPriority()[0]->hosts()[0], zone_b_endpoints); + + priority_state_manager.initializePriorityFor(zone_a_endpoints); + priority_state_manager.registerHostForPriority(hosts_added[1], zone_a_endpoints); + + // Update the cluster's priority set with the added hosts + priority_state_manager.updateClusterPrioritySet( + 0, std::move(priority_state_manager.priorityState()[0].first), hosts_added, absl::nullopt, + absl::nullopt); + + // Check that the P=0 host set has the added hosts, and the expected HostsPerLocality state + const HostVector& hosts = cluster.prioritySet().hostSetsPerPriority()[0]->hosts(); + const HostsPerLocality& hosts_per_locality = + cluster.prioritySet().hostSetsPerPriority()[0]->hostsPerLocality(); + + EXPECT_EQ(3UL, hosts.size()); + EXPECT_EQ(true, hosts_per_locality.hasLocalLocality()); + EXPECT_EQ(2UL, hosts_per_locality.get().size()); + + EXPECT_EQ(1UL, hosts_per_locality.get()[0].size()); + EXPECT_EQ(zone_a, hosts_per_locality.get()[0][0]->locality()); + + EXPECT_EQ(2UL, hosts_per_locality.get()[1].size()); + EXPECT_EQ(zone_b, hosts_per_locality.get()[1][0]->locality()); + EXPECT_EQ(zone_b, hosts_per_locality.get()[1][1]->locality()); +} + } // namespace } // namespace Upstream } // namespace Envoy diff --git a/test/common/upstream/utility.h b/test/common/upstream/utility.h index 0458f90a15db..29ec75e9c42b 100644 --- a/test/common/upstream/utility.h +++ b/test/common/upstream/utility.h @@ -115,6 +115,17 @@ inline HostSharedPtr makeTestHost(ClusterInfoConstSharedPtr cluster, const std:: status, time_source); } +inline HostSharedPtr makeTestHost(ClusterInfoConstSharedPtr cluster, const std::string& url, + TimeSource& time_source, + envoy::config::core::v3::Locality locality, uint32_t weight = 1, + uint32_t priority = 0, + Host::HealthStatus status = Host::HealthStatus::UNKNOWN) { + return std::make_shared( + cluster, "", Network::Utility::resolveUrl(url), nullptr, weight, locality, + envoy::config::endpoint::v3::Endpoint::HealthCheckConfig::default_instance(), priority, + status, time_source); +} + inline HostSharedPtr makeTestHost(ClusterInfoConstSharedPtr cluster, const std::string& url, const envoy::config::core::v3::Metadata& metadata, TimeSource& time_source, uint32_t weight = 1) { @@ -126,6 +137,17 @@ inline HostSharedPtr makeTestHost(ClusterInfoConstSharedPtr cluster, const std:: envoy::config::core::v3::UNKNOWN, time_source); } +inline HostSharedPtr makeTestHost(ClusterInfoConstSharedPtr cluster, const std::string& url, + const envoy::config::core::v3::Metadata& metadata, + envoy::config::core::v3::Locality locality, + TimeSource& time_source, uint32_t weight = 1) { + return std::make_shared( + cluster, "", Network::Utility::resolveUrl(url), + std::make_shared(metadata), weight, locality, + envoy::config::endpoint::v3::Endpoint::HealthCheckConfig::default_instance(), 0, + envoy::config::core::v3::UNKNOWN, time_source); +} + inline HostSharedPtr makeTestHost(ClusterInfoConstSharedPtr cluster, const std::string& url, const envoy::config::endpoint::v3::Endpoint::HealthCheckConfig& health_check_config, diff --git a/test/extensions/clusters/eds/eds_test.cc b/test/extensions/clusters/eds/eds_test.cc index 917de1fb1de6..314b95e35d35 100644 --- a/test/extensions/clusters/eds/eds_test.cc +++ b/test/extensions/clusters/eds/eds_test.cc @@ -1817,6 +1817,55 @@ TEST_F(EdsTest, EndpointLocality) { EXPECT_EQ(nullptr, cluster_->prioritySet().hostSetsPerPriority()[0]->localityWeights()); } +TEST_F(EdsTest, EndpointCombineDuplicateLocalities) { + envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment; + cluster_load_assignment.set_cluster_name("fare"); + + auto* endpoints1 = cluster_load_assignment.add_endpoints(); + auto* locality1 = endpoints1->mutable_locality(); + locality1->set_region("oceania"); + locality1->set_zone("hello"); + locality1->set_sub_zone("world"); + + auto* endpoints2 = cluster_load_assignment.add_endpoints(); + auto* locality2 = endpoints2->mutable_locality(); + locality2->set_region("oceania"); + locality2->set_zone("hello"); + locality2->set_sub_zone("world"); + + { + auto* endpoint_address = endpoints1->add_lb_endpoints() + ->mutable_endpoint() + ->mutable_address() + ->mutable_socket_address(); + endpoint_address->set_address("1.2.3.4"); + endpoint_address->set_port_value(80); + } + { + auto* endpoint_address = endpoints2->add_lb_endpoints() + ->mutable_endpoint() + ->mutable_address() + ->mutable_socket_address(); + endpoint_address->set_address("2.3.4.5"); + endpoint_address->set_port_value(80); + } + + initialize(); + doOnConfigUpdateVerifyNoThrow(cluster_load_assignment); + EXPECT_TRUE(initialized_); + + auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts(); + EXPECT_EQ(hosts.size(), 2); + for (int i = 0; i < 2; ++i) { + EXPECT_EQ(0, hosts[i]->priority()); + const auto& locality = hosts[i]->locality(); + EXPECT_EQ("oceania", locality.region()); + EXPECT_EQ("hello", locality.zone()); + EXPECT_EQ("world", locality.sub_zone()); + } + EXPECT_EQ(nullptr, cluster_->prioritySet().hostSetsPerPriority()[0]->localityWeights()); +} + // Validate that onConfigUpdate() updates the endpoint locality of an existing endpoint. TEST_F(EdsTest, EndpointLocalityUpdated) { envoy::config::endpoint::v3::ClusterLoadAssignment cluster_load_assignment; diff --git a/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc index 5de22a6d181e..e0cf7e3f06c8 100644 --- a/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc +++ b/test/extensions/load_balancing_policies/maglev/maglev_lb_test.cc @@ -370,8 +370,13 @@ TEST_P(MaglevLoadBalancerTest, Weighted) { // Locality weighted sanity test when localities have the same weights. Host weights for hosts in // different localities shouldn't matter. TEST_P(MaglevLoadBalancerTest, LocalityWeightedSameLocalityWeights) { - host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), 1), - makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), 2)}; + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + + host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), zone_a, 1), + makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), zone_b, 2)}; host_set_.healthy_hosts_ = host_set_.hosts_; host_set_.hosts_per_locality_ = makeHostsPerLocality({{host_set_.hosts_[0]}, {host_set_.hosts_[1]}}); @@ -413,9 +418,16 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedSameLocalityWeights) { // Locality weighted sanity test when localities have different weights. Host weights for hosts in // different localities shouldn't matter. TEST_P(MaglevLoadBalancerTest, LocalityWeightedDifferentLocalityWeights) { - host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), 1), - makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), 2), - makeTestHost(info_, "tcp://127.0.0.1:92", simTime(), 3)}; + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + + host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), zone_a, 1), + makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), zone_c, 2), + makeTestHost(info_, "tcp://127.0.0.1:92", simTime(), zone_b, 3)}; host_set_.healthy_hosts_ = host_set_.hosts_; host_set_.hosts_per_locality_ = makeHostsPerLocality({{host_set_.hosts_[0]}, {host_set_.hosts_[2]}, {host_set_.hosts_[1]}}); @@ -472,8 +484,13 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedAllZeroLocalityWeights) { // Validate that when we are in global panic and have localities, we get sane // results (fall back to non-healthy hosts). TEST_P(MaglevLoadBalancerTest, LocalityWeightedGlobalPanic) { - host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), 1), - makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), 2)}; + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + + host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), zone_a, 1), + makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), zone_b, 2)}; host_set_.healthy_hosts_ = {}; host_set_.hosts_per_locality_ = makeHostsPerLocality({{host_set_.hosts_[0]}, {host_set_.hosts_[1]}}); @@ -515,10 +532,16 @@ TEST_P(MaglevLoadBalancerTest, LocalityWeightedGlobalPanic) { // Given extremely lopsided locality weights, and a table that isn't large enough to fit all hosts, // expect that the least-weighted hosts appear once, and the most-weighted host fills the remainder. TEST_P(MaglevLoadBalancerTest, LocalityWeightedLopsided) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + host_set_.hosts_.clear(); HostVector heavy_but_sparse, light_but_dense; for (uint32_t i = 0; i < 1024; ++i) { - auto host(makeTestHost(info_, fmt::format("tcp://127.0.0.1:{}", i), simTime())); + auto host_locality = i == 0 ? zone_a : zone_b; + auto host(makeTestHost(info_, fmt::format("tcp://127.0.0.1:{}", i), simTime(), host_locality)); host_set_.hosts_.push_back(host); (i == 0 ? heavy_but_sparse : light_but_dense).push_back(host); } diff --git a/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc b/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc index 911bbc51e2b7..97f37b70d9be 100644 --- a/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc +++ b/test/extensions/load_balancing_policies/ring_hash/ring_hash_lb_test.cc @@ -639,8 +639,13 @@ TEST_P(RingHashLoadBalancerTest, HostWeightedLargeRing) { // Given locality weights all 0, expect the same behavior as if no hosts were provided at all. TEST_P(RingHashLoadBalancerTest, ZeroLocalityWeights) { - hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:91", simTime())}; + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + + hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), zone_b)}; hostSet().healthy_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality({{hostSet().hosts_[0]}, {hostSet().hosts_[1]}}); @@ -655,10 +660,19 @@ TEST_P(RingHashLoadBalancerTest, ZeroLocalityWeights) { // Given localities with weights 1, 2, 3 and 0, and a ring size of exactly 6, expect the correct // number of hashes for each host. TEST_P(RingHashLoadBalancerTest, LocalityWeightedTinyRing) { - hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:91", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:92", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:93", simTime())}; + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + envoy::config::core::v3::Locality zone_d; + zone_d.set_zone("D"); + + hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:92", simTime(), zone_c), + makeTestHost(info_, "tcp://127.0.0.1:93", simTime(), zone_d)}; hostSet().healthy_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0]}, {hostSet().hosts_[1]}, {hostSet().hosts_[2]}, {hostSet().hosts_[3]}}); @@ -690,10 +704,19 @@ TEST_P(RingHashLoadBalancerTest, LocalityWeightedTinyRing) { // Given localities with weights 1, 2, 3 and 0, and a sufficiently large ring, expect that requests // will distribute to the hosts with approximately the right proportion. TEST_P(RingHashLoadBalancerTest, LocalityWeightedLargeRing) { - hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:91", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:92", simTime()), - makeTestHost(info_, "tcp://127.0.0.1:93", simTime())}; + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + envoy::config::core::v3::Locality zone_c; + zone_c.set_zone("C"); + envoy::config::core::v3::Locality zone_d; + zone_d.set_zone("D"); + + hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), zone_a), + makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), zone_b), + makeTestHost(info_, "tcp://127.0.0.1:92", simTime(), zone_c), + makeTestHost(info_, "tcp://127.0.0.1:93", simTime(), zone_d)}; hostSet().healthy_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0]}, {hostSet().hosts_[1]}, {hostSet().hosts_[2]}, {hostSet().hosts_[3]}}); @@ -725,12 +748,17 @@ TEST_P(RingHashLoadBalancerTest, LocalityWeightedLargeRing) { // Given both host weights and locality weights, expect the correct number of hashes for each host. TEST_P(RingHashLoadBalancerTest, HostAndLocalityWeightedTinyRing) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + // :90 and :91 have a 1:2 ratio within the first locality, :92 and :93 have a 1:2 ratio within the // second locality, and the two localities have a 1:2 ratio overall. - hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), 1), - makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), 2), - makeTestHost(info_, "tcp://127.0.0.1:92", simTime(), 1), - makeTestHost(info_, "tcp://127.0.0.1:93", simTime(), 2)}; + hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), zone_a, 1), + makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), zone_a, 2), + makeTestHost(info_, "tcp://127.0.0.1:92", simTime(), zone_b, 1), + makeTestHost(info_, "tcp://127.0.0.1:93", simTime(), zone_b, 2)}; hostSet().healthy_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0], hostSet().hosts_[1]}, {hostSet().hosts_[2], hostSet().hosts_[3]}}); @@ -763,12 +791,17 @@ TEST_P(RingHashLoadBalancerTest, HostAndLocalityWeightedTinyRing) { // Given both host weights and locality weights, and a sufficiently large ring, expect that requests // will distribute to the hosts with approximately the right proportion. TEST_P(RingHashLoadBalancerTest, HostAndLocalityWeightedLargeRing) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + // :90 and :91 have a 1:2 ratio within the first locality, :92 and :93 have a 1:2 ratio within the // second locality, and the two localities have a 1:2 ratio overall. - hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), 1), - makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), 2), - makeTestHost(info_, "tcp://127.0.0.1:92", simTime(), 1), - makeTestHost(info_, "tcp://127.0.0.1:93", simTime(), 2)}; + hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90", simTime(), zone_a, 1), + makeTestHost(info_, "tcp://127.0.0.1:91", simTime(), zone_a, 2), + makeTestHost(info_, "tcp://127.0.0.1:92", simTime(), zone_b, 1), + makeTestHost(info_, "tcp://127.0.0.1:93", simTime(), zone_b, 2)}; hostSet().healthy_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0], hostSet().hosts_[1]}, {hostSet().hosts_[2], hostSet().hosts_[3]}}); @@ -872,10 +905,16 @@ TEST_P(RingHashLoadBalancerTest, LargeFractionalScale) { // Given extremely lopsided locality weights, and a ring that isn't large enough to fit all hosts, // expect that the correct proportion of hosts will be present in the ring. TEST_P(RingHashLoadBalancerTest, LopsidedWeightSmallScale) { + envoy::config::core::v3::Locality zone_a; + zone_a.set_zone("A"); + envoy::config::core::v3::Locality zone_b; + zone_b.set_zone("B"); + hostSet().hosts_.clear(); HostVector heavy_but_sparse, light_but_dense; for (uint32_t i = 0; i < 1024; ++i) { - auto host(makeTestHost(info_, fmt::format("tcp://127.0.0.1:{}", i), simTime())); + auto host_locality = i == 0 ? zone_a : zone_b; + auto host(makeTestHost(info_, fmt::format("tcp://127.0.0.1:{}", i), simTime(), host_locality)); hostSet().hosts_.push_back(host); (i == 0 ? heavy_but_sparse : light_but_dense).push_back(host); } diff --git a/test/server/admin/config_dump_handler_test.cc b/test/server/admin/config_dump_handler_test.cc index 962039e3c8e9..8d2dc83b6330 100644 --- a/test/server/admin/config_dump_handler_test.cc +++ b/test/server/admin/config_dump_handler_test.cc @@ -200,44 +200,44 @@ TEST_P(AdminInstanceTest, ConfigDumpWithLocalityEndpoint) { ON_CALL(*cluster.info_, addedViaApi()).WillByDefault(Return(false)); - Upstream::MockHostSet* host_set_1 = cluster.priority_set_.getMockHostSet(0); - auto host_1 = std::make_shared>(); - host_set_1->hosts_.emplace_back(host_1); + const std::string hostname_for_healthcheck = "test_hostname_healthcheck"; + const std::string empty_hostname_for_healthcheck = ""; envoy::config::core::v3::Locality locality_1; - locality_1.set_region("oceania"); - locality_1.set_zone("hello"); - locality_1.set_sub_zone("world"); - const std::string hostname_for_healthcheck = "test_hostname_healthcheck"; - const std::string hostname_1 = "foo.com"; + Upstream::MockHostSet* host_set_1 = cluster.priority_set_.getMockHostSet(0); + auto host_1 = std::make_shared>(); + host_set_1->hosts_.emplace_back(host_1); + const std::string hostname_1 = "coo.com"; - addHostInfo(*host_1, hostname_1, "tcp://1.2.3.4:80", locality_1, hostname_for_healthcheck, - "tcp://1.2.3.5:90", 5, 6); + addHostInfo(*host_1, hostname_1, "tcp://1.2.3.8:8", locality_1, empty_hostname_for_healthcheck, + "tcp://1.2.3.8:8", 3, 4); + const std::string hostname_2 = "foo.com"; auto host_2 = std::make_shared>(); host_set_1->hosts_.emplace_back(host_2); - const std::string empty_hostname_for_healthcheck = ""; - const std::string hostname_2 = "boo.com"; - - addHostInfo(*host_2, hostname_2, "tcp://1.2.3.7:8", locality_1, empty_hostname_for_healthcheck, - "tcp://1.2.3.7:8", 3, 6); envoy::config::core::v3::Locality locality_2; + locality_2.set_region("oceania"); + locality_2.set_zone("hello"); + locality_2.set_sub_zone("world"); + + addHostInfo(*host_2, hostname_2, "tcp://1.2.3.4:80", locality_2, hostname_for_healthcheck, + "tcp://1.2.3.5:90", 5, 6); auto host_3 = std::make_shared>(); host_set_1->hosts_.emplace_back(host_3); - const std::string hostname_3 = "coo.com"; + const std::string hostname_3 = "boo.com"; - addHostInfo(*host_3, hostname_3, "tcp://1.2.3.8:8", locality_2, empty_hostname_for_healthcheck, - "tcp://1.2.3.8:8", 3, 4); + addHostInfo(*host_3, hostname_3, "tcp://1.2.3.7:8", locality_2, empty_hostname_for_healthcheck, + "tcp://1.2.3.7:8", 3, 6); std::vector locality_hosts = { - {Upstream::HostSharedPtr(host_1), Upstream::HostSharedPtr(host_2)}, - {Upstream::HostSharedPtr(host_3)}}; + {Upstream::HostSharedPtr(host_1)}, + {Upstream::HostSharedPtr(host_2), Upstream::HostSharedPtr(host_3)}}; auto hosts_per_locality = new Upstream::HostsPerLocalityImpl(std::move(locality_hosts), false); - Upstream::LocalityWeightsConstSharedPtr locality_weights{new Upstream::LocalityWeights{1, 3}}; + Upstream::LocalityWeightsConstSharedPtr locality_weights{new Upstream::LocalityWeights{3, 1}}; ON_CALL(*host_set_1, hostsPerLocality()).WillByDefault(ReturnRef(*hosts_per_locality)); ON_CALL(*host_set_1, localityWeights()).WillByDefault(Return(locality_weights)); @@ -246,7 +246,7 @@ TEST_P(AdminInstanceTest, ConfigDumpWithLocalityEndpoint) { host_set_2->hosts_.emplace_back(host_4); const std::string hostname_4 = "doo.com"; - addHostInfo(*host_4, hostname_4, "tcp://1.2.3.9:8", locality_1, empty_hostname_for_healthcheck, + addHostInfo(*host_4, hostname_4, "tcp://1.2.3.9:8", locality_2, empty_hostname_for_healthcheck, "tcp://1.2.3.9:8", 3, 2); Buffer::OwnedImpl response; @@ -263,6 +263,28 @@ TEST_P(AdminInstanceTest, ConfigDumpWithLocalityEndpoint) { "@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment", "cluster_name": "fake_cluster", "endpoints": [ + { + "locality": {}, + "lb_endpoints": [ + { + "endpoint": { + "address": { + "socket_address": { + "address": "1.2.3.8", + "port_value": 8 + } + }, + "health_check_config": {}, + "hostname": "coo.com" + }, + "health_status": "HEALTHY", + "metadata": {}, + "load_balancing_weight": 3 + } + ], + "load_balancing_weight": 3, + "priority": 4 + }, { "locality": { "region": "oceania", @@ -307,28 +329,6 @@ TEST_P(AdminInstanceTest, ConfigDumpWithLocalityEndpoint) { "load_balancing_weight": 1, "priority": 6 }, - { - "locality": {}, - "lb_endpoints": [ - { - "endpoint": { - "address": { - "socket_address": { - "address": "1.2.3.8", - "port_value": 8 - } - }, - "health_check_config": {}, - "hostname": "coo.com" - }, - "health_status": "HEALTHY", - "metadata": {}, - "load_balancing_weight": 3 - } - ], - "load_balancing_weight": 3, - "priority": 4 - }, { "locality": { "region": "oceania",