From 055803866999c16dc45ccbc992cd3d4bc4a54f4f Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Thu, 26 Sep 2019 21:33:17 +0530 Subject: [PATCH 01/32] xds: make log levels consistent (#8320) Signed-off-by: Rama Chavali --- source/common/upstream/cds_api_impl.cc | 4 ++-- source/common/upstream/cluster_manager_impl.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index f6d93f3da6b4..838ec09721b7 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -76,7 +76,7 @@ void CdsApiImpl::onConfigUpdate( } if (cm_.addOrUpdateCluster(cluster, resource.version())) { any_applied = true; - ENVOY_LOG(debug, "cds: add/update cluster '{}'", cluster.name()); + ENVOY_LOG(info, "cds: add/update cluster '{}'", cluster.name()); } else { ENVOY_LOG(debug, "cds: add/update cluster '{}' skipped", cluster.name()); } @@ -87,7 +87,7 @@ void CdsApiImpl::onConfigUpdate( for (const auto& resource_name : removed_resources) { if (cm_.removeCluster(resource_name)) { any_applied = true; - ENVOY_LOG(debug, "cds: remove cluster '{}'", resource_name); + ENVOY_LOG(info, "cds: remove cluster '{}'", resource_name); } } diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index e5c06231c4ca..c873db74fcf5 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -529,7 +529,7 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::api::v2::Cluster& clust init_helper_.addCluster(*cluster_entry->cluster_); } else { auto& cluster_entry = warming_clusters_.at(cluster_name); - ENVOY_LOG(info, "add/update cluster {} starting warming", cluster_name); + ENVOY_LOG(debug, "add/update cluster {} starting warming", cluster_name); cluster_entry->cluster_->initialize([this, cluster_name] { auto warming_it = warming_clusters_.find(cluster_name); auto& cluster_entry = *warming_it->second; @@ -541,7 +541,7 @@ bool ClusterManagerImpl::addOrUpdateCluster(const envoy::api::v2::Cluster& clust active_clusters_[cluster_name] = std::move(warming_it->second); warming_clusters_.erase(warming_it); - ENVOY_LOG(info, "warming cluster {} complete", cluster_name); + ENVOY_LOG(debug, "warming cluster {} complete", cluster_name); createOrUpdateThreadLocalCluster(cluster_entry); onClusterInit(*cluster_entry.cluster_); updateClusterCounts(); From 51757dca1df458c78e63cccc6e7d0f357082bff0 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Thu, 26 Sep 2019 09:47:09 -0700 Subject: [PATCH 02/32] grpc: Move the ifdefs around the instantiation of GoogleRpcContext. (#8333) Signed-off-by: Joshua Marantz --- source/common/grpc/google_async_client_impl.h | 7 ++++++- source/common/grpc/google_grpc_context.cc | 4 ---- source/exe/main_common.h | 2 ++ source/extensions/transport_sockets/alts/config.cc | 8 ++++++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/source/common/grpc/google_async_client_impl.h b/source/common/grpc/google_async_client_impl.h index 688c846cb2df..00abc1d5a8f3 100644 --- a/source/common/grpc/google_async_client_impl.h +++ b/source/common/grpc/google_async_client_impl.h @@ -85,7 +85,12 @@ class GoogleAsyncClientThreadLocal : public ThreadLocal::ThreadLocalObject, private: void completionThread(); - // Instantiate this first to ensure grpc_init() is called. + // There is blanket google-grpc initialization in MainCommonBase, but that + // doesn't cover unit tests. However, putting blanket coverage in ProcessWide + // causes background threaded memory allocation in all unit tests making it + // hard to measure memory. Thus we also initialize grpc using our idempotent + // wrapper-class in classes that need it. See + // https://github.com/envoyproxy/envoy/issues/8282 for details. GoogleGrpcContext google_grpc_context_; // The CompletionQueue for in-flight operations. This must precede completion_thread_ to ensure it diff --git a/source/common/grpc/google_grpc_context.cc b/source/common/grpc/google_grpc_context.cc index 0a337ec590c9..b2b27ce8065a 100644 --- a/source/common/grpc/google_grpc_context.cc +++ b/source/common/grpc/google_grpc_context.cc @@ -15,9 +15,7 @@ namespace Grpc { GoogleGrpcContext::GoogleGrpcContext() : instance_tracker_(instanceTracker()) { Thread::LockGuard lock(instance_tracker_.mutex_); if (++instance_tracker_.live_instances_ == 1) { -#ifdef ENVOY_GOOGLE_GRPC grpc_init(); -#endif } } @@ -30,9 +28,7 @@ GoogleGrpcContext::~GoogleGrpcContext() { Thread::LockGuard lock(instance_tracker_.mutex_); ASSERT(instance_tracker_.live_instances_ > 0); if (--instance_tracker_.live_instances_ == 0) { -#ifdef ENVOY_GOOGLE_GRPC grpc_shutdown_blocking(); // Waiting for quiescence avoids non-determinism in tests. -#endif } } diff --git a/source/exe/main_common.h b/source/exe/main_common.h index 228f6e5e04a2..96a9e9dedf88 100644 --- a/source/exe/main_common.h +++ b/source/exe/main_common.h @@ -67,7 +67,9 @@ class MainCommonBase { protected: ProcessWide process_wide_; // Process-wide state setup/teardown (excluding grpc). +#ifdef ENVOY_GOOGLE_GRPC Grpc::GoogleGrpcContext google_grpc_context_; +#endif const Envoy::OptionsImpl& options_; Server::ComponentFactory& component_factory_; Thread::ThreadFactory& thread_factory_; diff --git a/source/extensions/transport_sockets/alts/config.cc b/source/extensions/transport_sockets/alts/config.cc index 65ca6297eb07..d4c65c882386 100644 --- a/source/extensions/transport_sockets/alts/config.cc +++ b/source/extensions/transport_sockets/alts/config.cc @@ -68,7 +68,15 @@ class AltsSharedState : public Singleton::Instance { ~AltsSharedState() override { grpc_alts_shared_resource_dedicated_shutdown(); } private: + // There is blanket google-grpc initialization in MainCommonBase, but that + // doesn't cover unit tests. However, putting blanket coverage in ProcessWide + // causes background threaded memory allocation in all unit tests making it + // hard to measure memory. Thus we also initialize grpc using our idempotent + // wrapper-class in classes that need it. See + // https://github.com/envoyproxy/envoy/issues/8282 for details. +#ifdef ENVOY_GOOGLE_GRPC Grpc::GoogleGrpcContext google_grpc_context_; +#endif }; SINGLETON_MANAGER_REGISTRATION(alts_shared_state); From 4b2dd0cba819c2b99dc865d752007a240fe06016 Mon Sep 17 00:00:00 2001 From: Kuat Date: Thu, 26 Sep 2019 10:58:33 -0700 Subject: [PATCH 03/32] async client: add hash_policy option (#8206) Description: Add hash policy to the async client. This is needed to support consistent hash load balancing for external requests originating from the proxy (e.g. telemetry, ext authz, WASM, etc). PR moves hash policy implementation to http namespace with no functional changes. Risk Level: low Testing: unit test Docs Changes: none Release Notes: none Fixes #4899 Signed-off-by: Kuat Yessenov --- include/envoy/http/BUILD | 11 ++ include/envoy/http/async_client.h | 16 +++ include/envoy/http/hash_policy.h | 44 +++++++ include/envoy/router/BUILD | 1 + include/envoy/router/router.h | 35 +---- source/common/http/BUILD | 11 ++ source/common/http/async_client_impl.cc | 3 +- source/common/http/async_client_impl.h | 19 ++- source/common/http/hash_policy.cc | 142 +++++++++++++++++++++ source/common/http/hash_policy.h | 41 ++++++ source/common/router/BUILD | 1 + source/common/router/config_impl.cc | 138 +------------------- source/common/router/config_impl.h | 39 +----- test/common/http/async_client_impl_test.cc | 44 ++++++- test/common/router/config_impl_test.cc | 2 +- test/common/router/router_test.cc | 6 +- test/mocks/router/mocks.h | 7 +- 17 files changed, 341 insertions(+), 219 deletions(-) create mode 100644 include/envoy/http/hash_policy.h create mode 100644 source/common/http/hash_policy.cc create mode 100644 source/common/http/hash_policy.h diff --git a/include/envoy/http/BUILD b/include/envoy/http/BUILD index ce03a17f9624..97477bfb77d0 100644 --- a/include/envoy/http/BUILD +++ b/include/envoy/http/BUILD @@ -15,6 +15,8 @@ envoy_cc_library( deps = [ ":message_interface", "//include/envoy/event:dispatcher_interface", + "//source/common/protobuf", + "@envoy_api//envoy/api/v2/route:route_cc", ], ) @@ -71,6 +73,15 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "hash_policy_interface", + hdrs = ["hash_policy.h"], + deps = [ + ":header_map_interface", + "//include/envoy/network:address_interface", + ], +) + envoy_cc_library( name = "header_map_interface", hdrs = ["header_map.h"], diff --git a/include/envoy/http/async_client.h b/include/envoy/http/async_client.h index c80eb38db6d4..54c2e3bed0e2 100644 --- a/include/envoy/http/async_client.h +++ b/include/envoy/http/async_client.h @@ -3,9 +3,12 @@ #include #include +#include "envoy/api/v2/route/route.pb.h" #include "envoy/event/dispatcher.h" #include "envoy/http/message.h" +#include "common/protobuf/protobuf.h" + #include "absl/types/optional.h" namespace Envoy { @@ -158,6 +161,11 @@ class AsyncClient { send_xff = v; return *this; } + StreamOptions& setHashPolicy( + const Protobuf::RepeatedPtrField& v) { + hash_policy = v; + return *this; + } // For gmock test bool operator==(const StreamOptions& src) const { @@ -177,6 +185,9 @@ class AsyncClient { // If true, x-forwarded-for header will be added. bool send_xff{true}; + + // Provides the hash policy for hashing load balancing strategies. + Protobuf::RepeatedPtrField hash_policy; }; /** @@ -199,6 +210,11 @@ class AsyncClient { StreamOptions::setSendXff(v); return *this; } + RequestOptions& setHashPolicy( + const Protobuf::RepeatedPtrField& v) { + StreamOptions::setHashPolicy(v); + return *this; + } // For gmock test bool operator==(const RequestOptions& src) const { return StreamOptions::operator==(src); } diff --git a/include/envoy/http/hash_policy.h b/include/envoy/http/hash_policy.h new file mode 100644 index 000000000000..aab762199950 --- /dev/null +++ b/include/envoy/http/hash_policy.h @@ -0,0 +1,44 @@ +#pragma once + +#include "envoy/http/header_map.h" +#include "envoy/network/address.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Http { + +/** + * Request hash policy. I.e., if using a hashing load balancer, how a request should be hashed onto + * an upstream host. + */ +class HashPolicy { +public: + virtual ~HashPolicy() = default; + + /** + * A callback used for requesting that a cookie be set with the given lifetime. + * @param key the name of the cookie to be set + * @param path the path of the cookie, or the empty string if no path should be set. + * @param ttl the lifetime of the cookie + * @return std::string the opaque value of the cookie that will be set + */ + using AddCookieCallback = std::function; + + /** + * @param downstream_address is the address of the connected client host, or nullptr if the + * request is initiated from within this host + * @param headers stores the HTTP headers for the stream + * @param add_cookie is called to add a set-cookie header on the reply sent to the downstream + * host + * @return absl::optional an optional hash value to route on. A hash value might not be + * returned if for example the specified HTTP header does not exist. + */ + virtual absl::optional + generateHash(const Network::Address::Instance* downstream_address, const HeaderMap& headers, + AddCookieCallback add_cookie) const PURE; +}; + +} // namespace Http +} // namespace Envoy diff --git a/include/envoy/router/BUILD b/include/envoy/router/BUILD index d62054044670..b7877db25c79 100644 --- a/include/envoy/router/BUILD +++ b/include/envoy/router/BUILD @@ -52,6 +52,7 @@ envoy_cc_library( "//include/envoy/config:typed_metadata_interface", "//include/envoy/http:codec_interface", "//include/envoy/http:codes_interface", + "//include/envoy/http:hash_policy_interface", "//include/envoy/http:header_map_interface", "//include/envoy/tracing:http_tracer_interface", "//include/envoy/upstream:resource_manager_interface", diff --git a/include/envoy/router/router.h b/include/envoy/router/router.h index dd365c1e772b..4719d5f29201 100644 --- a/include/envoy/router/router.h +++ b/include/envoy/router/router.h @@ -14,6 +14,7 @@ #include "envoy/config/typed_metadata.h" #include "envoy/http/codec.h" #include "envoy/http/codes.h" +#include "envoy/http/hash_policy.h" #include "envoy/http/header_map.h" #include "envoy/tracing/http_tracer.h" #include "envoy/upstream/resource_manager.h" @@ -431,38 +432,6 @@ class VirtualHost { virtual bool includeAttemptCount() const PURE; }; -/** - * Route hash policy. I.e., if using a hashing load balancer, how the route should be hashed onto - * an upstream host. - */ -class HashPolicy { -public: - virtual ~HashPolicy() = default; - - /** - * A callback used for requesting that a cookie be set with the given lifetime. - * @param key the name of the cookie to be set - * @param path the path of the cookie, or the empty string if no path should be set. - * @param ttl the lifetime of the cookie - * @return std::string the opaque value of the cookie that will be set - */ - using AddCookieCallback = std::function; - - /** - * @param downstream_address is the address of the connected client host, or nullptr if the - * request is initiated from within this host - * @param headers stores the HTTP headers for the stream - * @param add_cookie is called to add a set-cookie header on the reply sent to the downstream - * host - * @return absl::optional an optional hash value to route on. A hash value might not be - * returned if for example the specified HTTP header does not exist. - */ - virtual absl::optional - generateHash(const Network::Address::Instance* downstream_address, const Http::HeaderMap& headers, - AddCookieCallback add_cookie) const PURE; -}; - /** * Route level hedging policy. */ @@ -604,7 +573,7 @@ class RouteEntry : public ResponseEntry { /** * @return const HashPolicy* the optional hash policy for the route. */ - virtual const HashPolicy* hashPolicy() const PURE; + virtual const Http::HashPolicy* hashPolicy() const PURE; /** * @return const HedgePolicy& the hedge policy for the route. All routes have a hedge policy even diff --git a/source/common/http/BUILD b/source/common/http/BUILD index 04c882059daa..05553bd52916 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -213,6 +213,17 @@ envoy_cc_library( deps = ["//include/envoy/http:header_map_interface"], ) +envoy_cc_library( + name = "hash_policy_lib", + srcs = ["hash_policy.cc"], + hdrs = ["hash_policy.h"], + deps = [ + ":utility_lib", + "//include/envoy/http:hash_policy_interface", + "@envoy_api//envoy/api/v2/route:route_cc", + ], +) + envoy_cc_library( name = "header_map_lib", srcs = ["header_map_impl.cc"], diff --git a/source/common/http/async_client_impl.cc b/source/common/http/async_client_impl.cc index 054a2ef2dede..709d48cc319c 100644 --- a/source/common/http/async_client_impl.cc +++ b/source/common/http/async_client_impl.cc @@ -76,7 +76,8 @@ AsyncStreamImpl::AsyncStreamImpl(AsyncClientImpl& parent, AsyncClient::StreamCal : parent_(parent), stream_callbacks_(callbacks), stream_id_(parent.config_.random_.random()), router_(parent.config_), stream_info_(Protocol::Http11, parent.dispatcher().timeSource()), tracing_config_(Tracing::EgressConfig::get()), - route_(std::make_shared(parent_.cluster_->name(), options.timeout)), + route_(std::make_shared(parent_.cluster_->name(), options.timeout, + options.hash_policy)), send_xff_(options.send_xff) { if (options.buffer_body_for_retry) { buffered_body_ = std::make_unique(); diff --git a/source/common/http/async_client_impl.h b/source/common/http/async_client_impl.h index f00f89fa28d6..aa520f90c835 100644 --- a/source/common/http/async_client_impl.h +++ b/source/common/http/async_client_impl.h @@ -191,8 +191,14 @@ class AsyncStreamImpl : public AsyncClient::Stream, struct RouteEntryImpl : public Router::RouteEntry { RouteEntryImpl(const std::string& cluster_name, - const absl::optional& timeout) - : cluster_name_(cluster_name), timeout_(timeout) {} + const absl::optional& timeout, + const Protobuf::RepeatedPtrField& + hash_policy) + : cluster_name_(cluster_name), timeout_(timeout) { + if (!hash_policy.empty()) { + hash_policy_ = std::make_unique(hash_policy); + } + } // Router::RouteEntry const std::string& clusterName() const override { return cluster_name_; } @@ -203,7 +209,7 @@ class AsyncStreamImpl : public AsyncClient::Stream, void finalizeRequestHeaders(Http::HeaderMap&, const StreamInfo::StreamInfo&, bool) const override {} void finalizeResponseHeaders(Http::HeaderMap&, const StreamInfo::StreamInfo&) const override {} - const Router::HashPolicy* hashPolicy() const override { return nullptr; } + const HashPolicy* hashPolicy() const override { return hash_policy_.get(); } const Router::HedgePolicy& hedgePolicy() const override { return hedge_policy_; } const Router::MetadataMatchCriteria* metadataMatchCriteria() const override { return nullptr; } Upstream::ResourcePriority priority() const override { @@ -251,6 +257,7 @@ class AsyncStreamImpl : public AsyncClient::Stream, return Router::InternalRedirectAction::PassThrough; } const std::string& routeName() const override { return route_name_; } + std::unique_ptr hash_policy_; static const NullHedgePolicy hedge_policy_; static const NullRateLimitPolicy rate_limit_policy_; static const NullRetryPolicy retry_policy_; @@ -270,8 +277,10 @@ class AsyncStreamImpl : public AsyncClient::Stream, struct RouteImpl : public Router::Route { RouteImpl(const std::string& cluster_name, - const absl::optional& timeout) - : route_entry_(cluster_name, timeout) {} + const absl::optional& timeout, + const Protobuf::RepeatedPtrField& + hash_policy) + : route_entry_(cluster_name, timeout, hash_policy) {} // Router::Route const Router::DirectResponseEntry* directResponseEntry() const override { return nullptr; } diff --git a/source/common/http/hash_policy.cc b/source/common/http/hash_policy.cc new file mode 100644 index 000000000000..ec50a20194d1 --- /dev/null +++ b/source/common/http/hash_policy.cc @@ -0,0 +1,142 @@ +#include "common/http/hash_policy.h" + +#include "common/http/utility.h" + +namespace Envoy { +namespace Http { + +class HashMethodImplBase : public HashPolicyImpl::HashMethod { +public: + explicit HashMethodImplBase(bool terminal) : terminal_(terminal) {} + + bool terminal() const override { return terminal_; } + +private: + const bool terminal_; +}; + +class HeaderHashMethod : public HashMethodImplBase { +public: + HeaderHashMethod(const std::string& header_name, bool terminal) + : HashMethodImplBase(terminal), header_name_(header_name) {} + + absl::optional evaluate(const Network::Address::Instance*, const HeaderMap& headers, + const HashPolicy::AddCookieCallback) const override { + absl::optional hash; + + const HeaderEntry* header = headers.get(header_name_); + if (header) { + hash = HashUtil::xxHash64(header->value().getStringView()); + } + return hash; + } + +private: + const LowerCaseString header_name_; +}; + +class CookieHashMethod : public HashMethodImplBase { +public: + CookieHashMethod(const std::string& key, const std::string& path, + const absl::optional& ttl, bool terminal) + : HashMethodImplBase(terminal), key_(key), path_(path), ttl_(ttl) {} + + absl::optional evaluate(const Network::Address::Instance*, const HeaderMap& headers, + const HashPolicy::AddCookieCallback add_cookie) const override { + absl::optional hash; + std::string value = Utility::parseCookieValue(headers, key_); + if (value.empty() && ttl_.has_value()) { + value = add_cookie(key_, path_, ttl_.value()); + hash = HashUtil::xxHash64(value); + + } else if (!value.empty()) { + hash = HashUtil::xxHash64(value); + } + return hash; + } + +private: + const std::string key_; + const std::string path_; + const absl::optional ttl_; +}; + +class IpHashMethod : public HashMethodImplBase { +public: + IpHashMethod(bool terminal) : HashMethodImplBase(terminal) {} + + absl::optional evaluate(const Network::Address::Instance* downstream_addr, + const HeaderMap&, + const HashPolicy::AddCookieCallback) const override { + if (downstream_addr == nullptr) { + return absl::nullopt; + } + auto* downstream_ip = downstream_addr->ip(); + if (downstream_ip == nullptr) { + return absl::nullopt; + } + const auto& downstream_addr_str = downstream_ip->addressAsString(); + if (downstream_addr_str.empty()) { + return absl::nullopt; + } + return HashUtil::xxHash64(downstream_addr_str); + } +}; + +HashPolicyImpl::HashPolicyImpl( + absl::Span hash_policies) { + // TODO(htuch): Add support for cookie hash policies, #1295 + hash_impls_.reserve(hash_policies.size()); + + for (auto* hash_policy : hash_policies) { + switch (hash_policy->policy_specifier_case()) { + case envoy::api::v2::route::RouteAction::HashPolicy::kHeader: + hash_impls_.emplace_back( + new HeaderHashMethod(hash_policy->header().header_name(), hash_policy->terminal())); + break; + case envoy::api::v2::route::RouteAction::HashPolicy::kCookie: { + absl::optional ttl; + if (hash_policy->cookie().has_ttl()) { + ttl = std::chrono::seconds(hash_policy->cookie().ttl().seconds()); + } + hash_impls_.emplace_back(new CookieHashMethod(hash_policy->cookie().name(), + hash_policy->cookie().path(), ttl, + hash_policy->terminal())); + break; + } + case envoy::api::v2::route::RouteAction::HashPolicy::kConnectionProperties: + if (hash_policy->connection_properties().source_ip()) { + hash_impls_.emplace_back(new IpHashMethod(hash_policy->terminal())); + } + break; + default: + throw EnvoyException( + fmt::format("Unsupported hash policy {}", hash_policy->policy_specifier_case())); + } + } +} + +absl::optional +HashPolicyImpl::generateHash(const Network::Address::Instance* downstream_addr, + const HeaderMap& headers, const AddCookieCallback add_cookie) const { + absl::optional hash; + for (const HashMethodPtr& hash_impl : hash_impls_) { + const absl::optional new_hash = + hash_impl->evaluate(downstream_addr, headers, add_cookie); + if (new_hash) { + // Rotating the old value prevents duplicate hash rules from cancelling each other out + // and preserves all of the entropy + const uint64_t old_value = hash ? ((hash.value() << 1) | (hash.value() >> 63)) : 0; + hash = old_value ^ new_hash.value(); + } + // If the policy is a terminal policy and a hash has been generated, ignore + // the rest of the hash policies. + if (hash_impl->terminal() && hash) { + break; + } + } + return hash; +} + +} // namespace Http +} // namespace Envoy diff --git a/source/common/http/hash_policy.h b/source/common/http/hash_policy.h new file mode 100644 index 000000000000..9dd0f510f71e --- /dev/null +++ b/source/common/http/hash_policy.h @@ -0,0 +1,41 @@ +#pragma once + +#include "envoy/api/v2/route/route.pb.h" +#include "envoy/http/hash_policy.h" + +namespace Envoy { +namespace Http { + +/** + * Implementation of HashPolicy that reads from the proto route config and only currently supports + * hashing on an HTTP header. + */ +class HashPolicyImpl : public HashPolicy { +public: + explicit HashPolicyImpl( + absl::Span hash_policy); + + // Http::HashPolicy + absl::optional generateHash(const Network::Address::Instance* downstream_addr, + const HeaderMap& headers, + const AddCookieCallback add_cookie) const override; + + class HashMethod { + public: + virtual ~HashMethod() = default; + virtual absl::optional evaluate(const Network::Address::Instance* downstream_addr, + const HeaderMap& headers, + const AddCookieCallback add_cookie) const PURE; + + // If the method is a terminal method, ignore rest of the hash policy chain. + virtual bool terminal() const PURE; + }; + + using HashMethodPtr = std::unique_ptr; + +private: + std::vector hash_impls_; +}; + +} // namespace Http +} // namespace Envoy diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 24f78d68d4b5..c94382b5fb4e 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -43,6 +43,7 @@ envoy_cc_library( "//source/common/config:metadata_lib", "//source/common/config:rds_json_lib", "//source/common/config:well_known_names", + "//source/common/http:hash_policy_lib", "//source/common/http:header_utility_lib", "//source/common/http:headers_lib", "//source/common/http:utility_lib", diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index d98db7d8f171..7c620a365a27 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -188,142 +188,6 @@ ShadowPolicyImpl::ShadowPolicyImpl(const envoy::api::v2::route::RouteAction& con } } -class HashMethodImplBase : public HashPolicyImpl::HashMethod { -public: - explicit HashMethodImplBase(bool terminal) : terminal_(terminal) {} - - bool terminal() const override { return terminal_; } - -private: - const bool terminal_; -}; - -class HeaderHashMethod : public HashMethodImplBase { -public: - HeaderHashMethod(const std::string& header_name, bool terminal) - : HashMethodImplBase(terminal), header_name_(header_name) {} - - absl::optional evaluate(const Network::Address::Instance*, - const Http::HeaderMap& headers, - const HashPolicy::AddCookieCallback) const override { - absl::optional hash; - - const Http::HeaderEntry* header = headers.get(header_name_); - if (header) { - hash = HashUtil::xxHash64(header->value().getStringView()); - } - return hash; - } - -private: - const Http::LowerCaseString header_name_; -}; - -class CookieHashMethod : public HashMethodImplBase { -public: - CookieHashMethod(const std::string& key, const std::string& path, - const absl::optional& ttl, bool terminal) - : HashMethodImplBase(terminal), key_(key), path_(path), ttl_(ttl) {} - - absl::optional evaluate(const Network::Address::Instance*, - const Http::HeaderMap& headers, - const HashPolicy::AddCookieCallback add_cookie) const override { - absl::optional hash; - std::string value = Http::Utility::parseCookieValue(headers, key_); - if (value.empty() && ttl_.has_value()) { - value = add_cookie(key_, path_, ttl_.value()); - hash = HashUtil::xxHash64(value); - - } else if (!value.empty()) { - hash = HashUtil::xxHash64(value); - } - return hash; - } - -private: - const std::string key_; - const std::string path_; - const absl::optional ttl_; -}; - -class IpHashMethod : public HashMethodImplBase { -public: - IpHashMethod(bool terminal) : HashMethodImplBase(terminal) {} - - absl::optional evaluate(const Network::Address::Instance* downstream_addr, - const Http::HeaderMap&, - const HashPolicy::AddCookieCallback) const override { - if (downstream_addr == nullptr) { - return absl::nullopt; - } - auto* downstream_ip = downstream_addr->ip(); - if (downstream_ip == nullptr) { - return absl::nullopt; - } - const auto& downstream_addr_str = downstream_ip->addressAsString(); - if (downstream_addr_str.empty()) { - return absl::nullopt; - } - return HashUtil::xxHash64(downstream_addr_str); - } -}; - -HashPolicyImpl::HashPolicyImpl( - const Protobuf::RepeatedPtrField& - hash_policies) { - // TODO(htuch): Add support for cookie hash policies, #1295 - hash_impls_.reserve(hash_policies.size()); - - for (auto& hash_policy : hash_policies) { - switch (hash_policy.policy_specifier_case()) { - case envoy::api::v2::route::RouteAction::HashPolicy::kHeader: - hash_impls_.emplace_back( - new HeaderHashMethod(hash_policy.header().header_name(), hash_policy.terminal())); - break; - case envoy::api::v2::route::RouteAction::HashPolicy::kCookie: { - absl::optional ttl; - if (hash_policy.cookie().has_ttl()) { - ttl = std::chrono::seconds(hash_policy.cookie().ttl().seconds()); - } - hash_impls_.emplace_back(new CookieHashMethod( - hash_policy.cookie().name(), hash_policy.cookie().path(), ttl, hash_policy.terminal())); - break; - } - case envoy::api::v2::route::RouteAction::HashPolicy::kConnectionProperties: - if (hash_policy.connection_properties().source_ip()) { - hash_impls_.emplace_back(new IpHashMethod(hash_policy.terminal())); - } - break; - default: - throw EnvoyException( - fmt::format("Unsupported hash policy {}", hash_policy.policy_specifier_case())); - } - } -} - -absl::optional -HashPolicyImpl::generateHash(const Network::Address::Instance* downstream_addr, - const Http::HeaderMap& headers, - const AddCookieCallback add_cookie) const { - absl::optional hash; - for (const HashMethodPtr& hash_impl : hash_impls_) { - const absl::optional new_hash = - hash_impl->evaluate(downstream_addr, headers, add_cookie); - if (new_hash) { - // Rotating the old value prevents duplicate hash rules from cancelling each other out - // and preserves all of the entropy - const uint64_t old_value = hash ? ((hash.value() << 1) | (hash.value() >> 63)) : 0; - hash = old_value ^ new_hash.value(); - } - // If the policy is a terminal policy and a hash has been generated, ignore - // the rest of the hash policies. - if (hash_impl->terminal() && hash) { - break; - } - } - return hash; -} - DecoratorImpl::DecoratorImpl(const envoy::api::v2::route::Decorator& decorator) : operation_(decorator.operation()) {} @@ -456,7 +320,7 @@ RouteEntryImplBase::RouteEntryImplBase(const VirtualHostImpl& vhost, } if (!route.route().hash_policy().empty()) { - hash_policy_ = std::make_unique(route.route().hash_policy()); + hash_policy_ = std::make_unique(route.route().hash_policy()); } // Only set include_vh_rate_limits_ to true if the rate limit policy for the route is empty diff --git a/source/common/router/config_impl.h b/source/common/router/config_impl.h index 2870438511b7..e3f5301696d4 100644 --- a/source/common/router/config_impl.h +++ b/source/common/router/config_impl.h @@ -18,6 +18,7 @@ #include "envoy/upstream/cluster_manager.h" #include "common/config/metadata.h" +#include "common/http/hash_policy.h" #include "common/http/header_utility.h" #include "common/router/config_utility.h" #include "common/router/header_formatter.h" @@ -282,38 +283,6 @@ class ShadowPolicyImpl : public ShadowPolicy { envoy::type::FractionalPercent default_value_; }; -/** - * Implementation of HashPolicy that reads from the proto route config and only currently supports - * hashing on an HTTP header. - */ -class HashPolicyImpl : public HashPolicy { -public: - explicit HashPolicyImpl( - const Protobuf::RepeatedPtrField& - hash_policy); - - // Router::HashPolicy - absl::optional generateHash(const Network::Address::Instance* downstream_addr, - const Http::HeaderMap& headers, - const AddCookieCallback add_cookie) const override; - - class HashMethod { - public: - virtual ~HashMethod() = default; - virtual absl::optional evaluate(const Network::Address::Instance* downstream_addr, - const Http::HeaderMap& headers, - const AddCookieCallback add_cookie) const PURE; - - // If the method is a terminal method, ignore rest of the hash policy chain. - virtual bool terminal() const PURE; - }; - - using HashMethodPtr = std::unique_ptr; - -private: - std::vector hash_impls_; -}; - /** * Implementation of HedgePolicy that reads from the proto route or virtual host config. */ @@ -415,7 +384,7 @@ class RouteEntryImplBase : public RouteEntry, bool insert_envoy_original_path) const override; void finalizeResponseHeaders(Http::HeaderMap& headers, const StreamInfo::StreamInfo& stream_info) const override; - const HashPolicy* hashPolicy() const override { return hash_policy_.get(); } + const Http::HashPolicy* hashPolicy() const override { return hash_policy_.get(); } const HedgePolicy& hedgePolicy() const override { return hedge_policy_; } @@ -514,7 +483,7 @@ class RouteEntryImplBase : public RouteEntry, } const CorsPolicy* corsPolicy() const override { return parent_->corsPolicy(); } - const HashPolicy* hashPolicy() const override { return parent_->hashPolicy(); } + const Http::HashPolicy* hashPolicy() const override { return parent_->hashPolicy(); } const HedgePolicy& hedgePolicy() const override { return parent_->hedgePolicy(); } Upstream::ResourcePriority priority() const override { return parent_->priority(); } const RateLimitPolicy& rateLimitPolicy() const override { return parent_->rateLimitPolicy(); } @@ -679,7 +648,7 @@ class RouteEntryImplBase : public RouteEntry, UpgradeMap upgrade_map_; const uint64_t total_cluster_weight_; - std::unique_ptr hash_policy_; + std::unique_ptr hash_policy_; MetadataMatchCriteriaConstPtr metadata_match_criteria_; HeaderParserPtr request_headers_parser_; HeaderParserPtr response_headers_parser_; diff --git a/test/common/http/async_client_impl_test.cc b/test/common/http/async_client_impl_test.cc index bac7b1c2f1ab..66845e6ced11 100644 --- a/test/common/http/async_client_impl_test.cc +++ b/test/common/http/async_client_impl_test.cc @@ -154,6 +154,46 @@ TEST_F(AsyncClientImplTest, Basic) { .value()); } +TEST_F(AsyncClientImplTest, BasicHashPolicy) { + message_->body() = std::make_unique("test body"); + Buffer::Instance& data = *message_->body(); + + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](StreamDecoder& decoder, + ConnectionPool::Callbacks& callbacks) -> ConnectionPool::Cancellable* { + callbacks.onPoolReady(stream_encoder_, cm_.conn_pool_.host_, stream_info_); + response_decoder_ = &decoder; + return nullptr; + })); + EXPECT_CALL(cm_, httpConnPoolForCluster(_, _, _, _)) + .WillOnce( + Invoke([&](const std::string&, Upstream::ResourcePriority, Http::Protocol, + Upstream::LoadBalancerContext* context) -> Http::ConnectionPool::Instance* { + // this is the hash of :path header value "/" + EXPECT_EQ(16761507700594825962UL, context->computeHashKey().value()); + return &cm_.conn_pool_; + })); + + TestHeaderMapImpl copy(message_->headers()); + copy.addCopy("x-envoy-internal", "true"); + copy.addCopy("x-forwarded-for", "127.0.0.1"); + copy.addCopy(":scheme", "http"); + + EXPECT_CALL(stream_encoder_, encodeHeaders(HeaderMapEqualRef(©), false)); + EXPECT_CALL(stream_encoder_, encodeData(BufferEqual(&data), true)); + expectSuccess(200); + + AsyncClient::RequestOptions options; + Protobuf::RepeatedPtrField hash_policy; + hash_policy.Add()->mutable_header()->set_header_name(":path"); + options.setHashPolicy(hash_policy); + client_.send(std::move(message_), callbacks_, options); + + HeaderMapPtr response_headers(new TestHeaderMapImpl{{":status", "200"}}); + response_decoder_->decodeHeaders(std::move(response_headers), false); + response_decoder_->decodeData(data, true); +} + TEST_F(AsyncClientImplTest, Retry) { ON_CALL(runtime_.snapshot_, featureEnabled("upstream.use_retry", 100)) .WillByDefault(Return(true)); @@ -937,7 +977,9 @@ TEST_F(AsyncClientImplTest, DumpState) { // Must not be in anonymous namespace for friend to work. class AsyncClientImplRouteTest : public testing::Test { public: - AsyncStreamImpl::RouteImpl route_impl{"foo", absl::nullopt}; + AsyncStreamImpl::RouteImpl route_impl{ + "foo", absl::nullopt, + Protobuf::RepeatedPtrField()}; }; // Test the extended fake route that AsyncClient uses. diff --git a/test/common/router/config_impl_test.cc b/test/common/router/config_impl_test.cc index 1d252778f0da..12e21e3cb187 100644 --- a/test/common/router/config_impl_test.cc +++ b/test/common/router/config_impl_test.cc @@ -1765,7 +1765,7 @@ class RouterMatcherHashPolicyTest : public testing::Test, public ConfigImplTestB } envoy::api::v2::RouteConfiguration route_config_; - HashPolicy::AddCookieCallback add_cookie_nop_; + Http::HashPolicy::AddCookieCallback add_cookie_nop_; private: std::unique_ptr config_; diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 43c58971c191..544ec1062904 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -512,7 +512,7 @@ TEST_F(RouterTest, AddCookie) { std::string cookie_value; EXPECT_CALL(callbacks_.route_->route_entry_.hash_policy_, generateHash(_, _, _)) .WillOnce(Invoke([&](const Network::Address::Instance*, const Http::HeaderMap&, - const HashPolicy::AddCookieCallback add_cookie) { + const Http::HashPolicy::AddCookieCallback add_cookie) { cookie_value = add_cookie("foo", "", std::chrono::seconds(1337)); return absl::optional(10); })); @@ -560,7 +560,7 @@ TEST_F(RouterTest, AddCookieNoDuplicate) { EXPECT_CALL(callbacks_.route_->route_entry_.hash_policy_, generateHash(_, _, _)) .WillOnce(Invoke([&](const Network::Address::Instance*, const Http::HeaderMap&, - const HashPolicy::AddCookieCallback add_cookie) { + const Http::HashPolicy::AddCookieCallback add_cookie) { // this should be ignored add_cookie("foo", "", std::chrono::seconds(1337)); return absl::optional(10); @@ -608,7 +608,7 @@ TEST_F(RouterTest, AddMultipleCookies) { std::string choco_c, foo_c; EXPECT_CALL(callbacks_.route_->route_entry_.hash_policy_, generateHash(_, _, _)) .WillOnce(Invoke([&](const Network::Address::Instance*, const Http::HeaderMap&, - const HashPolicy::AddCookieCallback add_cookie) { + const Http::HashPolicy::AddCookieCallback add_cookie) { choco_c = add_cookie("choco", "", std::chrono::seconds(15)); foo_c = add_cookie("foo", "/path", std::chrono::seconds(1337)); return absl::optional(10); diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 8e4905183395..e1416736308f 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -12,6 +12,7 @@ #include "envoy/config/config_provider.h" #include "envoy/config/typed_metadata.h" #include "envoy/event/dispatcher.h" +#include "envoy/http/hash_policy.h" #include "envoy/local_info/local_info.h" #include "envoy/router/rds.h" #include "envoy/router/route_config_provider_manager.h" @@ -237,12 +238,12 @@ class MockVirtualHost : public VirtualHost { TestCorsPolicy cors_policy_; }; -class MockHashPolicy : public HashPolicy { +class MockHashPolicy : public Http::HashPolicy { public: MockHashPolicy(); ~MockHashPolicy() override; - // Router::HashPolicy + // Http::HashPolicy MOCK_CONST_METHOD3(generateHash, absl::optional(const Network::Address::Instance* downstream_address, const Http::HeaderMap& headers, @@ -286,7 +287,7 @@ class MockRouteEntry : public RouteEntry { bool insert_envoy_original_path)); MOCK_CONST_METHOD2(finalizeResponseHeaders, void(Http::HeaderMap& headers, const StreamInfo::StreamInfo& stream_info)); - MOCK_CONST_METHOD0(hashPolicy, const HashPolicy*()); + MOCK_CONST_METHOD0(hashPolicy, const Http::HashPolicy*()); MOCK_CONST_METHOD0(hedgePolicy, const HedgePolicy&()); MOCK_CONST_METHOD0(metadataMatchCriteria, const Router::MetadataMatchCriteria*()); MOCK_CONST_METHOD0(priority, Upstream::ResourcePriority()); From 85a44d985fbecc46ad53692fabc7ec8d5079b9fc Mon Sep 17 00:00:00 2001 From: "Ryan A. Chapman" Date: Thu, 26 Sep 2019 12:00:32 -0600 Subject: [PATCH 04/32] Update dependency and docs: jwt_verify_lib (add HS256 support) (#8385) Description: update jwt_verify_lib to support HS256 tokens, also update documentation to show that Envoy now supports HS256 as well as RS384 and RS512. Risk Level: low Testing: upstream unit tests Docs Changes: added Release Notes: none Signed-off-by: Ryan A. Chapman --- bazel/repository_locations.bzl | 8 ++++---- .../configuration/http/http_filters/jwt_authn_filter.rst | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index ffaca75a7862..ebc5c2889578 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -160,10 +160,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/msgpack/msgpack-c/releases/download/cpp-3.2.0/msgpack-3.2.0.tar.gz"], ), com_github_google_jwt_verify = dict( - sha256 = "2d57d336239d5fe36a03849ddbea1bff09a1720e1c4a46bbb9743c71732b0d43", - strip_prefix = "jwt_verify_lib-0f14d43f20381cfae0469cb2309b2e220c0f0ea3", - # 2019-07-08 - urls = ["https://github.com/google/jwt_verify_lib/archive/0f14d43f20381cfae0469cb2309b2e220c0f0ea3.tar.gz"], + sha256 = "38a93926f362a330a2a4489ed799c260df0bc305417e2bb44d6745671d9641d7", + strip_prefix = "jwt_verify_lib-7e3191b0dcb72835aa63e308a53b541e7fda5458", + # 2019-09-23 + urls = ["https://github.com/google/jwt_verify_lib/archive/7e3191b0dcb72835aa63e308a53b541e7fda5458.tar.gz"], ), com_github_nodejs_http_parser = dict( sha256 = "ef26268c54c8084d17654ba2ed5140bffeffd2a040a895ffb22a6cca3f6c613f", diff --git a/docs/root/configuration/http/http_filters/jwt_authn_filter.rst b/docs/root/configuration/http/http_filters/jwt_authn_filter.rst index 878adea96a48..2e1a217c2d96 100644 --- a/docs/root/configuration/http/http_filters/jwt_authn_filter.rst +++ b/docs/root/configuration/http/http_filters/jwt_authn_filter.rst @@ -8,7 +8,7 @@ This HTTP filter can be used to verify JSON Web Token (JWT). It will verify its JWKS is needed to verify JWT signatures. They can be specified in the filter config or can be fetched remotely from a JWKS server. .. attention:: - Only ES256 and RS256 are supported for the JWT alg. + ES256, HS256, RS256, RS384 and RS512 are supported for the JWT alg. Configuration ------------- From 3ebedeb708a23062332a6fcdf33b462b7070adba Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Thu, 26 Sep 2019 13:47:59 -0700 Subject: [PATCH 05/32] build: use image and toolchains from envoy-build-tools repo (#8367) Signed-off-by: Lizan Zhou --- .bazelrc | 2 +- .circleci/config.yml | 13 +- bazel/dependency_imports.bzl | 2 +- bazel/repositories.bzl | 1 + bazel/repository_locations.bzl | 13 +- bazel/toolchains/README.md | 13 - bazel/toolchains/configs/.gitignore | 2 - .../configs/clang/bazel_0.29.1/cc/BUILD | 149 --- .../cc/armeabi_cc_toolchain_config.bzl | 82 -- .../cc/builtin_include_directory_paths | 14 - .../bazel_0.29.1/cc/cc_toolchain_config.bzl | 1133 ----------------- .../clang/bazel_0.29.1/cc/cc_wrapper.sh | 25 - .../configs/clang/bazel_0.29.1/config/BUILD | 53 - .../clang_libcxx/bazel_0.29.1/cc/BUILD | 149 --- .../cc/armeabi_cc_toolchain_config.bzl | 82 -- .../cc/builtin_include_directory_paths | 12 - .../bazel_0.29.1/cc/cc_toolchain_config.bzl | 1133 ----------------- .../bazel_0.29.1/cc/cc_wrapper.sh | 25 - .../clang_libcxx/bazel_0.29.1/config/BUILD | 53 - .../configs/gcc/bazel_0.29.1/cc/BUILD | 148 --- .../cc/armeabi_cc_toolchain_config.bzl | 82 -- .../cc/builtin_include_directory_paths | 14 - .../bazel_0.29.1/cc/cc_toolchain_config.bzl | 1133 ----------------- .../configs/gcc/bazel_0.29.1/cc/cc_wrapper.sh | 25 - .../configs/gcc/bazel_0.29.1/config/BUILD | 53 - bazel/toolchains/configs/versions.bzl | 18 - bazel/toolchains/empty.bzl | 18 - bazel/toolchains/rbe_toolchains_config.bzl | 73 -- bazel/toolchains/regenerate.sh | 13 - ci/build_container/Dockerfile-centos | 7 - ci/build_container/Dockerfile-ubuntu | 6 - ci/build_container/README.md | 8 - ci/build_container/build_container_centos.sh | 34 - ci/build_container/build_container_common.sh | 17 - ci/build_container/build_container_ubuntu.sh | 69 - ci/build_container/docker_build.sh | 6 - ci/build_container/docker_push.sh | 71 -- ci/envoy_build_sha.sh | 2 +- ci/run_clang_tidy.sh | 2 +- ci/run_envoy_docker.sh | 2 +- 40 files changed, 17 insertions(+), 4740 deletions(-) delete mode 100644 bazel/toolchains/README.md delete mode 100644 bazel/toolchains/configs/.gitignore delete mode 100755 bazel/toolchains/configs/clang/bazel_0.29.1/cc/BUILD delete mode 100755 bazel/toolchains/configs/clang/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl delete mode 100755 bazel/toolchains/configs/clang/bazel_0.29.1/cc/builtin_include_directory_paths delete mode 100755 bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_toolchain_config.bzl delete mode 100755 bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_wrapper.sh delete mode 100644 bazel/toolchains/configs/clang/bazel_0.29.1/config/BUILD delete mode 100755 bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/BUILD delete mode 100755 bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl delete mode 100755 bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/builtin_include_directory_paths delete mode 100755 bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_toolchain_config.bzl delete mode 100755 bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_wrapper.sh delete mode 100644 bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/config/BUILD delete mode 100755 bazel/toolchains/configs/gcc/bazel_0.29.1/cc/BUILD delete mode 100755 bazel/toolchains/configs/gcc/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl delete mode 100755 bazel/toolchains/configs/gcc/bazel_0.29.1/cc/builtin_include_directory_paths delete mode 100755 bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_toolchain_config.bzl delete mode 100755 bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_wrapper.sh delete mode 100644 bazel/toolchains/configs/gcc/bazel_0.29.1/config/BUILD delete mode 100644 bazel/toolchains/configs/versions.bzl delete mode 100644 bazel/toolchains/empty.bzl delete mode 100644 bazel/toolchains/rbe_toolchains_config.bzl delete mode 100755 bazel/toolchains/regenerate.sh delete mode 100644 ci/build_container/Dockerfile-centos delete mode 100644 ci/build_container/Dockerfile-ubuntu delete mode 100644 ci/build_container/README.md delete mode 100755 ci/build_container/build_container_centos.sh delete mode 100755 ci/build_container/build_container_common.sh delete mode 100755 ci/build_container/build_container_ubuntu.sh delete mode 100755 ci/build_container/docker_build.sh delete mode 100755 ci/build_container/docker_push.sh diff --git a/.bazelrc b/.bazelrc index fd60cc9b0dd9..8ac3842d1ab1 100644 --- a/.bazelrc +++ b/.bazelrc @@ -129,7 +129,7 @@ build:remote-clang --config=remote build:remote-clang --config=rbe-toolchain-clang # Docker sandbox -build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f +build:docker-sandbox --experimental_docker_image=envoyproxy/envoy-build@sha256:1500b75cb9bc2184a26f72f63631e926cd05d6ee628e6fd005f069d2f9756ac7 build:docker-sandbox --spawn_strategy=docker build:docker-sandbox --strategy=Javac=docker build:docker-sandbox --strategy=Closure=docker diff --git a/.circleci/config.yml b/.circleci/config.yml index ec4c45c62a1d..5db1d0de6ec9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,8 +4,8 @@ executors: ubuntu-build: description: "A regular build executor based on ubuntu image" docker: - # NOTE: Update bazel/toolchains/rbe_toolchains_config.bzl with sha256 digest to match the image here. - - image: envoyproxy/envoy-build:cb15cc3d2010aff77d6e022ddf6d723fa8becbc0 + # NOTE: Update this from https://github.com/envoyproxy/envoy-build-tools/blob/master/toolchains/rbe_toolchains_config.bzl#L7 + - image: envoyproxy/envoy-build-ubuntu@sha256:1500b75cb9bc2184a26f72f63631e926cd05d6ee628e6fd005f069d2f9756ac7 resource_class: xlarge working_directory: /source @@ -103,14 +103,6 @@ jobs: ci/do_circle_ci.sh bazel.clang_tidy no_output_timeout: 60m - build_image: - docker: - - image: google/cloud-sdk - steps: - - run: rm -rf /home/circleci/project/.git # CircleCI git caching is likely broken - - checkout - - setup_remote_docker - - run: ci/build_container/docker_push.sh docs: executor: ubuntu-build steps: @@ -143,7 +135,6 @@ workflows: - coverage_publish: requires: [coverage] - clang_tidy - - build_image - docs: filters: tags: diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index c7eb3e1cb702..e4c3e8e46f99 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -1,6 +1,6 @@ load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies") load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") -load("@envoy//bazel/toolchains:rbe_toolchains_config.bzl", "rbe_toolchains_config") +load("@envoy_build_tools//toolchains:rbe_toolchains_config.bzl", "rbe_toolchains_config") load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") # go version for rules_go diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index efc585e97d0a..95e59466b58c 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -157,6 +157,7 @@ def envoy_dependencies(skip_targets = []): _repository_impl("com_googlesource_code_re2") _com_google_cel_cpp() _repository_impl("bazel_toolchains") + _repository_impl("envoy_build_tools") _python_deps() _cc_deps() diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index ebc5c2889578..07674ee1bf08 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -4,13 +4,18 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.18.1/bazel-gazelle-0.18.1.tar.gz"], ), bazel_toolchains = dict( - sha256 = "ab0d8aaeaeeef413ddb03922dbdb99bbae9e1b2c157a87c77d70d45a830be5b0", - strip_prefix = "bazel-toolchains-0.29.1", + sha256 = "a1e273b6159ae858f53046f5bab9678cffa82a72f0bf0c0a9e4af8fddb91209c", + strip_prefix = "bazel-toolchains-0.29.6", urls = [ - "https://github.com/bazelbuild/bazel-toolchains/releases/download/0.29.1/bazel-toolchains-0.29.1.tar.gz", - "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/0.29.1.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/releases/download/0.29.6/bazel-toolchains-0.29.6.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/0.29.6.tar.gz", ], ), + envoy_build_tools = dict( + sha256 = "87e0968792b139b462621f5bd56c03126bc06ae4d408f1e956749f12fcc090d9", + strip_prefix = "envoy-build-tools-4433e52437af6936d0af95ebc3b16b4b6df38618", + urls = ["https://github.com/envoyproxy/envoy-build-tools/archive/4433e52437af6936d0af95ebc3b16b4b6df38618.tar.gz"], + ), boringssl = dict( sha256 = "c712766ddc844de2a38e686e1cdd7288795e9a6fe7f699c6636f1b76703db84e", strip_prefix = "boringssl-265728decec4370cd02b941f72fba9f0735e2923", diff --git a/bazel/toolchains/README.md b/bazel/toolchains/README.md deleted file mode 100644 index 38e01600e86e..000000000000 --- a/bazel/toolchains/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Bazel Toolchains - -This directory contains toolchains config generated for Bazel [RBE](https://docs.bazel.build/versions/master/remote-execution.html) and -[Docker sandbox](https://docs.bazel.build/versions/master/remote-execution-sandbox.html). - -To regenerate toolchain configs, update the docker image information in `rbe_toolchains_config.bzl` and run following command in an -environment with the latest Bazel and Docker installed: - -``` -bazel/toolchains/regenerate.sh -``` - -This will generate configs in `bazel/toolchains/configs`, check in those files so they can be used in CI. diff --git a/bazel/toolchains/configs/.gitignore b/bazel/toolchains/configs/.gitignore deleted file mode 100644 index 9683e8aa4d54..000000000000 --- a/bazel/toolchains/configs/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# RBE autoconfig generator will generate a .bazelrc but we don't need it. -.latest.bazelrc diff --git a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/BUILD b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/BUILD deleted file mode 100755 index da902b38d578..000000000000 --- a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/BUILD +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes. - -package(default_visibility = ["//visibility:public"]) - -load(":cc_toolchain_config.bzl", "cc_toolchain_config") -load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config") -load("@rules_cc//cc:defs.bzl", "cc_toolchain", "cc_toolchain_suite") - -licenses(["notice"]) # Apache 2.0 - -cc_library( - name = "malloc", -) - -filegroup( - name = "empty", - srcs = [], -) - -filegroup( - name = "cc_wrapper", - srcs = ["cc_wrapper.sh"], -) - -filegroup( - name = "compiler_deps", - srcs = glob(["extra_tools/**"], allow_empty = True) + [":builtin_include_directory_paths"], -) - -# This is the entry point for --crosstool_top. Toolchains are found -# by lopping off the name of --crosstool_top and searching for -# the "${CPU}" entry in the toolchains attribute. -cc_toolchain_suite( - name = "toolchain", - toolchains = { - "k8|clang": ":cc-compiler-k8", - "k8": ":cc-compiler-k8", - "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a", - "armeabi-v7a": ":cc-compiler-armeabi-v7a", - }, -) - -cc_toolchain( - name = "cc-compiler-k8", - toolchain_identifier = "local", - toolchain_config = ":local", - all_files = ":compiler_deps", - ar_files = ":compiler_deps", - as_files = ":compiler_deps", - compiler_files = ":compiler_deps", - dwp_files = ":empty", - linker_files = ":compiler_deps", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 1, -) - -cc_toolchain_config( - name = "local", - cpu = "k8", - compiler = "clang", - toolchain_identifier = "local", - host_system_name = "local", - target_system_name = "local", - target_libc = "local", - abi_version = "local", - abi_libc_version = "local", - cxx_builtin_include_directories = ["/usr/local/include", - "/usr/lib/llvm-8/lib/clang/8.0.1/include", - "/usr/include/x86_64-linux-gnu", - "/usr/include", - "/usr/include/c++/7.4.0", - "/usr/include/x86_64-linux-gnu/c++/7.4.0", - "/usr/include/c++/7.4.0/backward", - "/usr/include/clang/8.0.1/include"], - tool_paths = {"ar": "/usr/bin/ar", - "ld": "/usr/bin/ld", - "cpp": "/usr/bin/cpp", - "gcc": "/usr/lib/llvm-8/bin/clang", - "dwp": "/usr/bin/dwp", - "gcov": "/usr/lib/llvm-8/bin/llvm-profdata", - "nm": "/usr/bin/nm", - "objcopy": "/usr/bin/objcopy", - "objdump": "/usr/bin/objdump", - "strip": "/usr/bin/strip"}, - compile_flags = ["-U_FORTIFY_SOURCE", - "-fstack-protector", - "-Wall", - "-Wthread-safety", - "-Wself-assign", - "-fcolor-diagnostics", - "-fno-omit-frame-pointer"], - opt_compile_flags = ["-g0", - "-O2", - "-D_FORTIFY_SOURCE=1", - "-DNDEBUG", - "-ffunction-sections", - "-fdata-sections"], - dbg_compile_flags = ["-g"], - cxx_flags = ["-std=c++0x"], - link_flags = ["-fuse-ld=/usr/bin/ld.gold", - "-Wl,-no-as-needed", - "-Wl,-z,relro,-z,now", - "-B/usr/lib/llvm-8/bin", - "-lm", - "-fuse-ld=lld"], - link_libs = ["-l:libstdc++.a"], - opt_link_flags = ["-Wl,--gc-sections"], - unfiltered_compile_flags = ["-no-canonical-prefixes", - "-Wno-builtin-macro-redefined", - "-D__DATE__=\"redacted\"", - "-D__TIMESTAMP__=\"redacted\"", - "-D__TIME__=\"redacted\""], - coverage_compile_flags = ["-fprofile-instr-generate", "-fcoverage-mapping"], - coverage_link_flags = ["-fprofile-instr-generate"], - supports_start_end_lib = True, -) - -# Android tooling requires a default toolchain for the armeabi-v7a cpu. -cc_toolchain( - name = "cc-compiler-armeabi-v7a", - toolchain_identifier = "stub_armeabi-v7a", - toolchain_config = ":stub_armeabi-v7a", - all_files = ":empty", - ar_files = ":empty", - as_files = ":empty", - compiler_files = ":empty", - dwp_files = ":empty", - linker_files = ":empty", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 1, -) - -armeabi_cc_toolchain_config(name = "stub_armeabi-v7a") diff --git a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl deleted file mode 100755 index 94e0720bf6c9..000000000000 --- a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A Starlark cc_toolchain configuration rule""" - -load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "feature", - "tool_path", -) - -def _impl(ctx): - toolchain_identifier = "stub_armeabi-v7a" - host_system_name = "armeabi-v7a" - target_system_name = "armeabi-v7a" - target_cpu = "armeabi-v7a" - target_libc = "armeabi-v7a" - compiler = "compiler" - abi_version = "armeabi-v7a" - abi_libc_version = "armeabi-v7a" - cc_target_os = None - builtin_sysroot = None - action_configs = [] - - supports_pic_feature = feature(name = "supports_pic", enabled = True) - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - features = [supports_dynamic_linker_feature, supports_pic_feature] - - cxx_builtin_include_directories = [] - artifact_name_patterns = [] - make_variables = [] - - tool_paths = [ - tool_path(name = "ar", path = "/bin/false"), - tool_path(name = "compat-ld", path = "/bin/false"), - tool_path(name = "cpp", path = "/bin/false"), - tool_path(name = "dwp", path = "/bin/false"), - tool_path(name = "gcc", path = "/bin/false"), - tool_path(name = "gcov", path = "/bin/false"), - tool_path(name = "ld", path = "/bin/false"), - tool_path(name = "nm", path = "/bin/false"), - tool_path(name = "objcopy", path = "/bin/false"), - tool_path(name = "objdump", path = "/bin/false"), - tool_path(name = "strip", path = "/bin/false"), - ] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - artifact_name_patterns = artifact_name_patterns, - cxx_builtin_include_directories = cxx_builtin_include_directories, - toolchain_identifier = toolchain_identifier, - host_system_name = host_system_name, - target_system_name = target_system_name, - target_cpu = target_cpu, - target_libc = target_libc, - compiler = compiler, - abi_version = abi_version, - abi_libc_version = abi_libc_version, - tool_paths = tool_paths, - make_variables = make_variables, - builtin_sysroot = builtin_sysroot, - cc_target_os = cc_target_os, - ) - -armeabi_cc_toolchain_config = rule( - implementation = _impl, - attrs = {}, - provides = [CcToolchainConfigInfo], -) diff --git a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/builtin_include_directory_paths b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/builtin_include_directory_paths deleted file mode 100755 index 151e3b008a2b..000000000000 --- a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/builtin_include_directory_paths +++ /dev/null @@ -1,14 +0,0 @@ -This file is generated by cc_configure and contains builtin include directories -that /usr/lib/llvm-8/bin/clang reported. This file is a dependency of every compilation action and -changes to it will be reflected in the action cache key. When some of these -paths change, Bazel will make sure to rerun the action, even though none of -declared action inputs or the action commandline changes. - -/usr/local/include -/usr/lib/llvm-8/lib/clang/8.0.1/include -/usr/include/x86_64-linux-gnu -/usr/include -/usr/include/c++/7.4.0 -/usr/include/x86_64-linux-gnu/c++/7.4.0 -/usr/include/c++/7.4.0/backward -/usr/include/clang/8.0.1/include diff --git a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_toolchain_config.bzl deleted file mode 100755 index bf4d83940f89..000000000000 --- a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_toolchain_config.bzl +++ /dev/null @@ -1,1133 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A Starlark cc_toolchain configuration rule""" - -load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "feature", - "feature_set", - "flag_group", - "flag_set", - "tool_path", - "variable_with_value", - "with_feature_set", -) -load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") - -all_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ACTION_NAMES.lto_backend, -] - -all_cpp_compile_actions = [ - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, -] - -preprocessor_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, -] - -codegen_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, -] - -all_link_actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, -] - -lto_index_actions = [ - ACTION_NAMES.lto_index_for_executable, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, -] - -def _impl(ctx): - tool_paths = [ - tool_path(name = name, path = path) - for name, path in ctx.attr.tool_paths.items() - ] - action_configs = [] - - supports_pic_feature = feature( - name = "supports_pic", - enabled = True, - ) - supports_start_end_lib_feature = feature( - name = "supports_start_end_lib", - enabled = True, - ) - - default_compile_flags_feature = feature( - name = "default_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.compile_flags, - ), - ] if ctx.attr.compile_flags else []), - ), - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.dbg_compile_flags, - ), - ] if ctx.attr.dbg_compile_flags else []), - with_features = [with_feature_set(features = ["dbg"])], - ), - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.opt_compile_flags, - ), - ] if ctx.attr.opt_compile_flags else []), - with_features = [with_feature_set(features = ["opt"])], - ), - flag_set( - actions = all_cpp_compile_actions + [ACTION_NAMES.lto_backend], - flag_groups = ([ - flag_group( - flags = ctx.attr.cxx_flags, - ), - ] if ctx.attr.cxx_flags else []), - ), - ], - ) - - default_link_flags_feature = feature( - name = "default_link_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.link_flags, - ), - ] if ctx.attr.link_flags else []), - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.opt_link_flags, - ), - ] if ctx.attr.opt_link_flags else []), - with_features = [with_feature_set(features = ["opt"])], - ), - ], - ) - - dbg_feature = feature(name = "dbg") - - opt_feature = feature(name = "opt") - - sysroot_feature = feature( - name = "sysroot", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ] + all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["--sysroot=%{sysroot}"], - expand_if_available = "sysroot", - ), - ], - ), - ], - ) - - fdo_optimize_feature = feature( - name = "fdo_optimize", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-use=%{fdo_profile_path}", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - - user_compile_flags_feature = feature( - name = "user_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group( - flags = ["%{user_compile_flags}"], - iterate_over = "user_compile_flags", - expand_if_available = "user_compile_flags", - ), - ], - ), - ], - ) - - unfiltered_compile_flags_feature = feature( - name = "unfiltered_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.unfiltered_compile_flags, - ), - ] if ctx.attr.unfiltered_compile_flags else []), - ), - ], - ) - - library_search_directories_feature = feature( - name = "library_search_directories", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-L%{library_search_directories}"], - iterate_over = "library_search_directories", - expand_if_available = "library_search_directories", - ), - ], - ), - ], - ) - - static_libgcc_feature = feature( - name = "static_libgcc", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.lto_index_for_executable, - ACTION_NAMES.lto_index_for_dynamic_library, - ], - flag_groups = [flag_group(flags = ["-static-libgcc"])], - with_features = [ - with_feature_set(features = ["static_link_cpp_runtimes"]), - ], - ), - ], - ) - - pic_feature = feature( - name = "pic", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = [ - flag_group(flags = ["-fPIC"], expand_if_available = "pic"), - ], - ), - ], - ) - - per_object_debug_info_feature = feature( - name = "per_object_debug_info", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_codegen, - ], - flag_groups = [ - flag_group( - flags = ["-gsplit-dwarf"], - expand_if_available = "per_object_debug_info_file", - ), - ], - ), - ], - ) - - preprocessor_defines_feature = feature( - name = "preprocessor_defines", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["-D%{preprocessor_defines}"], - iterate_over = "preprocessor_defines", - ), - ], - ), - ], - ) - - cs_fdo_optimize_feature = feature( - name = "cs_fdo_optimize", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.lto_backend], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-use=%{fdo_profile_path}", - "-Xclang-only=-Wno-profile-instr-unprofiled", - "-Xclang-only=-Wno-profile-instr-out-of-date", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["csprofile"], - ) - - autofdo_feature = feature( - name = "autofdo", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [ - flag_group( - flags = [ - "-fauto-profile=%{fdo_profile_path}", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - runtime_library_search_directories_feature = feature( - name = "runtime_library_search_directories", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - iterate_over = "runtime_library_search_directories", - flag_groups = [ - flag_group( - flags = [ - "-Wl,-rpath,$EXEC_ORIGIN/%{runtime_library_search_directories}", - ], - expand_if_true = "is_cc_test", - ), - flag_group( - flags = [ - "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", - ], - expand_if_false = "is_cc_test", - ), - ], - expand_if_available = - "runtime_library_search_directories", - ), - ], - with_features = [ - with_feature_set(features = ["static_link_cpp_runtimes"]), - ], - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - iterate_over = "runtime_library_search_directories", - flag_groups = [ - flag_group( - flags = [ - "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", - ], - ), - ], - expand_if_available = - "runtime_library_search_directories", - ), - ], - with_features = [ - with_feature_set( - not_features = ["static_link_cpp_runtimes"], - ), - ], - ), - ], - ) - - fission_support_feature = feature( - name = "fission_support", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-Wl,--gdb-index"], - expand_if_available = "is_using_fission", - ), - ], - ), - ], - ) - - shared_flag_feature = feature( - name = "shared_flag", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, - ], - flag_groups = [flag_group(flags = ["-shared"])], - ), - ], - ) - - random_seed_feature = feature( - name = "random_seed", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = [ - flag_group( - flags = ["-frandom-seed=%{output_file}"], - expand_if_available = "output_file", - ), - ], - ), - ], - ) - - includes_feature = feature( - name = "includes", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ], - flag_groups = [ - flag_group( - flags = ["-include", "%{includes}"], - iterate_over = "includes", - expand_if_available = "includes", - ), - ], - ), - ], - ) - - fdo_instrument_feature = feature( - name = "fdo_instrument", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ] + all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = [ - "-fprofile-generate=%{fdo_instrument_path}", - "-fno-data-sections", - ], - expand_if_available = "fdo_instrument_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - cs_fdo_instrument_feature = feature( - name = "cs_fdo_instrument", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.lto_backend, - ] + all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = [ - "-fcs-profile-generate=%{cs_fdo_instrument_path}", - ], - expand_if_available = "cs_fdo_instrument_path", - ), - ], - ), - ], - provides = ["csprofile"], - ) - - include_paths_feature = feature( - name = "include_paths", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ], - flag_groups = [ - flag_group( - flags = ["-iquote", "%{quote_include_paths}"], - iterate_over = "quote_include_paths", - ), - flag_group( - flags = ["-I%{include_paths}"], - iterate_over = "include_paths", - ), - flag_group( - flags = ["-isystem", "%{system_include_paths}"], - iterate_over = "system_include_paths", - ), - ], - ), - ], - ) - - symbol_counts_feature = feature( - name = "symbol_counts", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = [ - "-Wl,--print-symbol-counts=%{symbol_counts_output}", - ], - expand_if_available = "symbol_counts_output", - ), - ], - ), - ], - ) - - llvm_coverage_map_format_feature = feature( - name = "llvm_coverage_map_format", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-instr-generate", - "-fcoverage-mapping", - ], - ), - ], - ), - flag_set( - actions = all_link_actions + lto_index_actions + [ - "objc-executable", - "objc++-executable", - ], - flag_groups = [ - flag_group(flags = ["-fprofile-instr-generate"]), - ], - ), - ], - requires = [feature_set(features = ["coverage"])], - provides = ["profile"], - ) - - strip_debug_symbols_feature = feature( - name = "strip_debug_symbols", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-Wl,-S"], - expand_if_available = "strip_debug_symbols", - ), - ], - ), - ], - ) - - build_interface_libraries_feature = feature( - name = "build_interface_libraries", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = [ - "%{generate_interface_library}", - "%{interface_library_builder_path}", - "%{interface_library_input_path}", - "%{interface_library_output_path}", - ], - expand_if_available = "generate_interface_library", - ), - ], - with_features = [ - with_feature_set( - features = ["supports_interface_shared_libraries"], - ), - ], - ), - ], - ) - - libraries_to_link_feature = feature( - name = "libraries_to_link", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - iterate_over = "libraries_to_link", - flag_groups = [ - flag_group( - flags = ["-Wl,--start-lib"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - flag_group( - flags = ["-Wl,-whole-archive"], - expand_if_true = - "libraries_to_link.is_whole_archive", - ), - flag_group( - flags = ["%{libraries_to_link.object_files}"], - iterate_over = "libraries_to_link.object_files", - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "interface_library", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "static_library", - ), - ), - flag_group( - flags = ["-l%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "dynamic_library", - ), - ), - flag_group( - flags = ["-l:%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "versioned_dynamic_library", - ), - ), - flag_group( - flags = ["-Wl,-no-whole-archive"], - expand_if_true = "libraries_to_link.is_whole_archive", - ), - flag_group( - flags = ["-Wl,--end-lib"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - ], - expand_if_available = "libraries_to_link", - ), - flag_group( - flags = ["-Wl,@%{thinlto_param_file}"], - expand_if_true = "thinlto_param_file", - ), - ], - ), - ], - ) - - user_link_flags_feature = feature( - name = "user_link_flags", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["%{user_link_flags}"], - iterate_over = "user_link_flags", - expand_if_available = "user_link_flags", - ), - ] + ([flag_group(flags = ctx.attr.link_libs)] if ctx.attr.link_libs else []), - ), - ], - ) - - fdo_prefetch_hints_feature = feature( - name = "fdo_prefetch_hints", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.lto_backend, - ], - flag_groups = [ - flag_group( - flags = [ - "-Xclang-only=-mllvm", - "-Xclang-only=-prefetch-hints-file=%{fdo_prefetch_hints_path}", - ], - expand_if_available = "fdo_prefetch_hints_path", - ), - ], - ), - ], - ) - - linkstamps_feature = feature( - name = "linkstamps", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["%{linkstamp_paths}"], - iterate_over = "linkstamp_paths", - expand_if_available = "linkstamp_paths", - ), - ], - ), - ], - ) - - gcc_coverage_map_format_feature = feature( - name = "gcc_coverage_map_format", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - "objc-executable", - "objc++-executable", - ], - flag_groups = [ - flag_group( - flags = ["-fprofile-arcs", "-ftest-coverage"], - expand_if_available = "gcov_gcno_file", - ), - ], - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [flag_group(flags = ["--coverage"])], - ), - ], - requires = [feature_set(features = ["coverage"])], - provides = ["profile"], - ) - - archiver_flags_feature = feature( - name = "archiver_flags", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.cpp_link_static_library], - flag_groups = [ - flag_group(flags = ["rcsD"]), - flag_group( - flags = ["%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - flag_set( - actions = [ACTION_NAMES.cpp_link_static_library], - flag_groups = [ - flag_group( - iterate_over = "libraries_to_link", - flag_groups = [ - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file", - ), - ), - flag_group( - flags = ["%{libraries_to_link.object_files}"], - iterate_over = "libraries_to_link.object_files", - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - ], - expand_if_available = "libraries_to_link", - ), - ], - ), - ], - ) - - force_pic_flags_feature = feature( - name = "force_pic_flags", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.lto_index_for_executable, - ], - flag_groups = [ - flag_group( - flags = ["-pie"], - expand_if_available = "force_pic", - ), - ], - ), - ], - ) - - dependency_file_feature = feature( - name = "dependency_file", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["-MD", "-MF", "%{dependency_file}"], - expand_if_available = "dependency_file", - ), - ], - ), - ], - ) - - dynamic_library_linker_tool_path = tool_paths - dynamic_library_linker_tool_feature = feature( - name = "dynamic_library_linker_tool", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = [" + cppLinkDynamicLibraryToolPath + "], - expand_if_available = "generate_interface_library", - ), - ], - with_features = [ - with_feature_set( - features = ["supports_interface_shared_libraries"], - ), - ], - ), - ], - ) - - output_execpath_flags_feature = feature( - name = "output_execpath_flags", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-o", "%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - ], - ) - - # Note that we also set --coverage for c++-link-nodeps-dynamic-library. The - # generated code contains references to gcov symbols, and the dynamic linker - # can't resolve them unless the library is linked against gcov. - coverage_feature = feature( - name = "coverage", - provides = ["profile"], - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = ([ - flag_group(flags = ctx.attr.coverage_compile_flags), - ] if ctx.attr.coverage_compile_flags else []), - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = ([ - flag_group(flags = ctx.attr.coverage_link_flags), - ] if ctx.attr.coverage_link_flags else []), - ), - ], - ) - - is_linux = ctx.attr.target_libc != "macosx" - - # TODO(#8303): Mac crosstool should also declare every feature. - if is_linux: - features = [ - dependency_file_feature, - random_seed_feature, - pic_feature, - per_object_debug_info_feature, - preprocessor_defines_feature, - includes_feature, - include_paths_feature, - fdo_instrument_feature, - cs_fdo_instrument_feature, - cs_fdo_optimize_feature, - fdo_prefetch_hints_feature, - autofdo_feature, - build_interface_libraries_feature, - dynamic_library_linker_tool_feature, - symbol_counts_feature, - shared_flag_feature, - linkstamps_feature, - output_execpath_flags_feature, - runtime_library_search_directories_feature, - library_search_directories_feature, - archiver_flags_feature, - force_pic_flags_feature, - fission_support_feature, - strip_debug_symbols_feature, - coverage_feature, - supports_pic_feature, - ] + ( - [ - supports_start_end_lib_feature, - ] if ctx.attr.supports_start_end_lib else [] - ) + [ - default_compile_flags_feature, - default_link_flags_feature, - libraries_to_link_feature, - user_link_flags_feature, - static_libgcc_feature, - fdo_optimize_feature, - supports_dynamic_linker_feature, - dbg_feature, - opt_feature, - user_compile_flags_feature, - sysroot_feature, - unfiltered_compile_flags_feature, - ] - else: - features = [ - supports_pic_feature, - ] + ( - [ - supports_start_end_lib_feature, - ] if ctx.attr.supports_start_end_lib else [] - ) + [ - coverage_feature, - default_compile_flags_feature, - default_link_flags_feature, - fdo_optimize_feature, - supports_dynamic_linker_feature, - dbg_feature, - opt_feature, - user_compile_flags_feature, - sysroot_feature, - unfiltered_compile_flags_feature, - ] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - cxx_builtin_include_directories = ctx.attr.cxx_builtin_include_directories, - toolchain_identifier = ctx.attr.toolchain_identifier, - host_system_name = ctx.attr.host_system_name, - target_system_name = ctx.attr.target_system_name, - target_cpu = ctx.attr.cpu, - target_libc = ctx.attr.target_libc, - compiler = ctx.attr.compiler, - abi_version = ctx.attr.abi_version, - abi_libc_version = ctx.attr.abi_libc_version, - tool_paths = tool_paths, - ) - -cc_toolchain_config = rule( - implementation = _impl, - attrs = { - "cpu": attr.string(mandatory = True), - "compiler": attr.string(mandatory = True), - "toolchain_identifier": attr.string(mandatory = True), - "host_system_name": attr.string(mandatory = True), - "target_system_name": attr.string(mandatory = True), - "target_libc": attr.string(mandatory = True), - "abi_version": attr.string(mandatory = True), - "abi_libc_version": attr.string(mandatory = True), - "cxx_builtin_include_directories": attr.string_list(), - "tool_paths": attr.string_dict(), - "compile_flags": attr.string_list(), - "dbg_compile_flags": attr.string_list(), - "opt_compile_flags": attr.string_list(), - "cxx_flags": attr.string_list(), - "link_flags": attr.string_list(), - "link_libs": attr.string_list(), - "opt_link_flags": attr.string_list(), - "unfiltered_compile_flags": attr.string_list(), - "coverage_compile_flags": attr.string_list(), - "coverage_link_flags": attr.string_list(), - "supports_start_end_lib": attr.bool(), - }, - provides = [CcToolchainConfigInfo], -) diff --git a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_wrapper.sh b/bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_wrapper.sh deleted file mode 100755 index b7ff6355883c..000000000000 --- a/bazel/toolchains/configs/clang/bazel_0.29.1/cc/cc_wrapper.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Ship the environment to the C++ action -# -set -eu - -# Set-up the environment - - -# Call the C++ compiler -/usr/lib/llvm-8/bin/clang "$@" diff --git a/bazel/toolchains/configs/clang/bazel_0.29.1/config/BUILD b/bazel/toolchains/configs/clang/bazel_0.29.1/config/BUILD deleted file mode 100644 index 5fb181617f25..000000000000 --- a/bazel/toolchains/configs/clang/bazel_0.29.1/config/BUILD +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is auto-generated by an rbe_autoconfig repository rule -# and should not be modified directly. -# See @bazel_toolchains//rules:rbe_repo.bzl - -package(default_visibility = ["//visibility:public"]) - -toolchain( - name = "cc-toolchain", - exec_compatible_with = [ - "@bazel_tools//platforms:x86_64", - "@bazel_tools//platforms:linux", - "@bazel_tools//tools/cpp:clang", - ], - target_compatible_with = [ - "@bazel_tools//platforms:linux", - "@bazel_tools//platforms:x86_64", - ], - toolchain = "//bazel/toolchains/configs/clang/bazel_0.29.1/cc:cc-compiler-k8", - toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", -) - -platform( - name = "platform", - constraint_values = [ - "@bazel_tools//platforms:x86_64", - "@bazel_tools//platforms:linux", - "@bazel_tools//tools/cpp:clang", - ], - remote_execution_properties = """ - properties: { - name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" - } - properties { - name: "OSFamily" - value: "Linux" - } - """, -) diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/BUILD b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/BUILD deleted file mode 100755 index 625db858205b..000000000000 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/BUILD +++ /dev/null @@ -1,149 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes. - -package(default_visibility = ["//visibility:public"]) - -load(":cc_toolchain_config.bzl", "cc_toolchain_config") -load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config") -load("@rules_cc//cc:defs.bzl", "cc_toolchain", "cc_toolchain_suite") - -licenses(["notice"]) # Apache 2.0 - -cc_library( - name = "malloc", -) - -filegroup( - name = "empty", - srcs = [], -) - -filegroup( - name = "cc_wrapper", - srcs = ["cc_wrapper.sh"], -) - -filegroup( - name = "compiler_deps", - srcs = glob(["extra_tools/**"], allow_empty = True) + [":builtin_include_directory_paths"], -) - -# This is the entry point for --crosstool_top. Toolchains are found -# by lopping off the name of --crosstool_top and searching for -# the "${CPU}" entry in the toolchains attribute. -cc_toolchain_suite( - name = "toolchain", - toolchains = { - "k8|clang": ":cc-compiler-k8", - "k8": ":cc-compiler-k8", - "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a", - "armeabi-v7a": ":cc-compiler-armeabi-v7a", - }, -) - -cc_toolchain( - name = "cc-compiler-k8", - toolchain_identifier = "local", - toolchain_config = ":local", - all_files = ":compiler_deps", - ar_files = ":compiler_deps", - as_files = ":compiler_deps", - compiler_files = ":compiler_deps", - dwp_files = ":empty", - linker_files = ":compiler_deps", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 1, -) - -cc_toolchain_config( - name = "local", - cpu = "k8", - compiler = "clang", - toolchain_identifier = "local", - host_system_name = "local", - target_system_name = "local", - target_libc = "local", - abi_version = "local", - abi_libc_version = "local", - cxx_builtin_include_directories = ["/usr/local/include", - "/usr/lib/llvm-8/lib/clang/8.0.1/include", - "/usr/include/x86_64-linux-gnu", - "/usr/include", - "/usr/lib/llvm-8/include/c++/v1", - "/usr/include/clang/8.0.1/include"], - tool_paths = {"ar": "/usr/bin/ar", - "ld": "/usr/bin/ld", - "cpp": "/usr/bin/cpp", - "gcc": "/usr/lib/llvm-8/bin/clang", - "dwp": "/usr/bin/dwp", - "gcov": "/usr/lib/llvm-8/bin/llvm-profdata", - "nm": "/usr/bin/nm", - "objcopy": "/usr/bin/objcopy", - "objdump": "/usr/bin/objdump", - "strip": "/usr/bin/strip"}, - compile_flags = ["-U_FORTIFY_SOURCE", - "-fstack-protector", - "-Wall", - "-Wthread-safety", - "-Wself-assign", - "-fcolor-diagnostics", - "-fno-omit-frame-pointer"], - opt_compile_flags = ["-g0", - "-O2", - "-D_FORTIFY_SOURCE=1", - "-DNDEBUG", - "-ffunction-sections", - "-fdata-sections"], - dbg_compile_flags = ["-g"], - cxx_flags = ["-stdlib=libc++"], - link_flags = ["-fuse-ld=/usr/bin/ld.gold", - "-Wl,-no-as-needed", - "-Wl,-z,relro,-z,now", - "-B/usr/lib/llvm-8/bin", - "-lm", - "-pthread", - "-fuse-ld=lld"], - link_libs = ["-l:libc++.a", - "-l:libc++abi.a"], - opt_link_flags = ["-Wl,--gc-sections"], - unfiltered_compile_flags = ["-no-canonical-prefixes", - "-Wno-builtin-macro-redefined", - "-D__DATE__=\"redacted\"", - "-D__TIMESTAMP__=\"redacted\"", - "-D__TIME__=\"redacted\""], - coverage_compile_flags = ["-fprofile-instr-generate", "-fcoverage-mapping"], - coverage_link_flags = ["-fprofile-instr-generate"], - supports_start_end_lib = True, -) - -# Android tooling requires a default toolchain for the armeabi-v7a cpu. -cc_toolchain( - name = "cc-compiler-armeabi-v7a", - toolchain_identifier = "stub_armeabi-v7a", - toolchain_config = ":stub_armeabi-v7a", - all_files = ":empty", - ar_files = ":empty", - as_files = ":empty", - compiler_files = ":empty", - dwp_files = ":empty", - linker_files = ":empty", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 1, -) - -armeabi_cc_toolchain_config(name = "stub_armeabi-v7a") diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl deleted file mode 100755 index 94e0720bf6c9..000000000000 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A Starlark cc_toolchain configuration rule""" - -load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "feature", - "tool_path", -) - -def _impl(ctx): - toolchain_identifier = "stub_armeabi-v7a" - host_system_name = "armeabi-v7a" - target_system_name = "armeabi-v7a" - target_cpu = "armeabi-v7a" - target_libc = "armeabi-v7a" - compiler = "compiler" - abi_version = "armeabi-v7a" - abi_libc_version = "armeabi-v7a" - cc_target_os = None - builtin_sysroot = None - action_configs = [] - - supports_pic_feature = feature(name = "supports_pic", enabled = True) - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - features = [supports_dynamic_linker_feature, supports_pic_feature] - - cxx_builtin_include_directories = [] - artifact_name_patterns = [] - make_variables = [] - - tool_paths = [ - tool_path(name = "ar", path = "/bin/false"), - tool_path(name = "compat-ld", path = "/bin/false"), - tool_path(name = "cpp", path = "/bin/false"), - tool_path(name = "dwp", path = "/bin/false"), - tool_path(name = "gcc", path = "/bin/false"), - tool_path(name = "gcov", path = "/bin/false"), - tool_path(name = "ld", path = "/bin/false"), - tool_path(name = "nm", path = "/bin/false"), - tool_path(name = "objcopy", path = "/bin/false"), - tool_path(name = "objdump", path = "/bin/false"), - tool_path(name = "strip", path = "/bin/false"), - ] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - artifact_name_patterns = artifact_name_patterns, - cxx_builtin_include_directories = cxx_builtin_include_directories, - toolchain_identifier = toolchain_identifier, - host_system_name = host_system_name, - target_system_name = target_system_name, - target_cpu = target_cpu, - target_libc = target_libc, - compiler = compiler, - abi_version = abi_version, - abi_libc_version = abi_libc_version, - tool_paths = tool_paths, - make_variables = make_variables, - builtin_sysroot = builtin_sysroot, - cc_target_os = cc_target_os, - ) - -armeabi_cc_toolchain_config = rule( - implementation = _impl, - attrs = {}, - provides = [CcToolchainConfigInfo], -) diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/builtin_include_directory_paths b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/builtin_include_directory_paths deleted file mode 100755 index d809f97268c1..000000000000 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/builtin_include_directory_paths +++ /dev/null @@ -1,12 +0,0 @@ -This file is generated by cc_configure and contains builtin include directories -that /usr/lib/llvm-8/bin/clang reported. This file is a dependency of every compilation action and -changes to it will be reflected in the action cache key. When some of these -paths change, Bazel will make sure to rerun the action, even though none of -declared action inputs or the action commandline changes. - -/usr/local/include -/usr/lib/llvm-8/lib/clang/8.0.1/include -/usr/include/x86_64-linux-gnu -/usr/include -/usr/lib/llvm-8/include/c++/v1 -/usr/include/clang/8.0.1/include diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_toolchain_config.bzl deleted file mode 100755 index bf4d83940f89..000000000000 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_toolchain_config.bzl +++ /dev/null @@ -1,1133 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A Starlark cc_toolchain configuration rule""" - -load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "feature", - "feature_set", - "flag_group", - "flag_set", - "tool_path", - "variable_with_value", - "with_feature_set", -) -load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") - -all_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ACTION_NAMES.lto_backend, -] - -all_cpp_compile_actions = [ - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, -] - -preprocessor_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, -] - -codegen_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, -] - -all_link_actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, -] - -lto_index_actions = [ - ACTION_NAMES.lto_index_for_executable, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, -] - -def _impl(ctx): - tool_paths = [ - tool_path(name = name, path = path) - for name, path in ctx.attr.tool_paths.items() - ] - action_configs = [] - - supports_pic_feature = feature( - name = "supports_pic", - enabled = True, - ) - supports_start_end_lib_feature = feature( - name = "supports_start_end_lib", - enabled = True, - ) - - default_compile_flags_feature = feature( - name = "default_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.compile_flags, - ), - ] if ctx.attr.compile_flags else []), - ), - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.dbg_compile_flags, - ), - ] if ctx.attr.dbg_compile_flags else []), - with_features = [with_feature_set(features = ["dbg"])], - ), - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.opt_compile_flags, - ), - ] if ctx.attr.opt_compile_flags else []), - with_features = [with_feature_set(features = ["opt"])], - ), - flag_set( - actions = all_cpp_compile_actions + [ACTION_NAMES.lto_backend], - flag_groups = ([ - flag_group( - flags = ctx.attr.cxx_flags, - ), - ] if ctx.attr.cxx_flags else []), - ), - ], - ) - - default_link_flags_feature = feature( - name = "default_link_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.link_flags, - ), - ] if ctx.attr.link_flags else []), - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.opt_link_flags, - ), - ] if ctx.attr.opt_link_flags else []), - with_features = [with_feature_set(features = ["opt"])], - ), - ], - ) - - dbg_feature = feature(name = "dbg") - - opt_feature = feature(name = "opt") - - sysroot_feature = feature( - name = "sysroot", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ] + all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["--sysroot=%{sysroot}"], - expand_if_available = "sysroot", - ), - ], - ), - ], - ) - - fdo_optimize_feature = feature( - name = "fdo_optimize", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-use=%{fdo_profile_path}", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - - user_compile_flags_feature = feature( - name = "user_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group( - flags = ["%{user_compile_flags}"], - iterate_over = "user_compile_flags", - expand_if_available = "user_compile_flags", - ), - ], - ), - ], - ) - - unfiltered_compile_flags_feature = feature( - name = "unfiltered_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.unfiltered_compile_flags, - ), - ] if ctx.attr.unfiltered_compile_flags else []), - ), - ], - ) - - library_search_directories_feature = feature( - name = "library_search_directories", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-L%{library_search_directories}"], - iterate_over = "library_search_directories", - expand_if_available = "library_search_directories", - ), - ], - ), - ], - ) - - static_libgcc_feature = feature( - name = "static_libgcc", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.lto_index_for_executable, - ACTION_NAMES.lto_index_for_dynamic_library, - ], - flag_groups = [flag_group(flags = ["-static-libgcc"])], - with_features = [ - with_feature_set(features = ["static_link_cpp_runtimes"]), - ], - ), - ], - ) - - pic_feature = feature( - name = "pic", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = [ - flag_group(flags = ["-fPIC"], expand_if_available = "pic"), - ], - ), - ], - ) - - per_object_debug_info_feature = feature( - name = "per_object_debug_info", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_codegen, - ], - flag_groups = [ - flag_group( - flags = ["-gsplit-dwarf"], - expand_if_available = "per_object_debug_info_file", - ), - ], - ), - ], - ) - - preprocessor_defines_feature = feature( - name = "preprocessor_defines", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["-D%{preprocessor_defines}"], - iterate_over = "preprocessor_defines", - ), - ], - ), - ], - ) - - cs_fdo_optimize_feature = feature( - name = "cs_fdo_optimize", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.lto_backend], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-use=%{fdo_profile_path}", - "-Xclang-only=-Wno-profile-instr-unprofiled", - "-Xclang-only=-Wno-profile-instr-out-of-date", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["csprofile"], - ) - - autofdo_feature = feature( - name = "autofdo", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [ - flag_group( - flags = [ - "-fauto-profile=%{fdo_profile_path}", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - runtime_library_search_directories_feature = feature( - name = "runtime_library_search_directories", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - iterate_over = "runtime_library_search_directories", - flag_groups = [ - flag_group( - flags = [ - "-Wl,-rpath,$EXEC_ORIGIN/%{runtime_library_search_directories}", - ], - expand_if_true = "is_cc_test", - ), - flag_group( - flags = [ - "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", - ], - expand_if_false = "is_cc_test", - ), - ], - expand_if_available = - "runtime_library_search_directories", - ), - ], - with_features = [ - with_feature_set(features = ["static_link_cpp_runtimes"]), - ], - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - iterate_over = "runtime_library_search_directories", - flag_groups = [ - flag_group( - flags = [ - "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", - ], - ), - ], - expand_if_available = - "runtime_library_search_directories", - ), - ], - with_features = [ - with_feature_set( - not_features = ["static_link_cpp_runtimes"], - ), - ], - ), - ], - ) - - fission_support_feature = feature( - name = "fission_support", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-Wl,--gdb-index"], - expand_if_available = "is_using_fission", - ), - ], - ), - ], - ) - - shared_flag_feature = feature( - name = "shared_flag", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, - ], - flag_groups = [flag_group(flags = ["-shared"])], - ), - ], - ) - - random_seed_feature = feature( - name = "random_seed", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = [ - flag_group( - flags = ["-frandom-seed=%{output_file}"], - expand_if_available = "output_file", - ), - ], - ), - ], - ) - - includes_feature = feature( - name = "includes", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ], - flag_groups = [ - flag_group( - flags = ["-include", "%{includes}"], - iterate_over = "includes", - expand_if_available = "includes", - ), - ], - ), - ], - ) - - fdo_instrument_feature = feature( - name = "fdo_instrument", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ] + all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = [ - "-fprofile-generate=%{fdo_instrument_path}", - "-fno-data-sections", - ], - expand_if_available = "fdo_instrument_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - cs_fdo_instrument_feature = feature( - name = "cs_fdo_instrument", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.lto_backend, - ] + all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = [ - "-fcs-profile-generate=%{cs_fdo_instrument_path}", - ], - expand_if_available = "cs_fdo_instrument_path", - ), - ], - ), - ], - provides = ["csprofile"], - ) - - include_paths_feature = feature( - name = "include_paths", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ], - flag_groups = [ - flag_group( - flags = ["-iquote", "%{quote_include_paths}"], - iterate_over = "quote_include_paths", - ), - flag_group( - flags = ["-I%{include_paths}"], - iterate_over = "include_paths", - ), - flag_group( - flags = ["-isystem", "%{system_include_paths}"], - iterate_over = "system_include_paths", - ), - ], - ), - ], - ) - - symbol_counts_feature = feature( - name = "symbol_counts", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = [ - "-Wl,--print-symbol-counts=%{symbol_counts_output}", - ], - expand_if_available = "symbol_counts_output", - ), - ], - ), - ], - ) - - llvm_coverage_map_format_feature = feature( - name = "llvm_coverage_map_format", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-instr-generate", - "-fcoverage-mapping", - ], - ), - ], - ), - flag_set( - actions = all_link_actions + lto_index_actions + [ - "objc-executable", - "objc++-executable", - ], - flag_groups = [ - flag_group(flags = ["-fprofile-instr-generate"]), - ], - ), - ], - requires = [feature_set(features = ["coverage"])], - provides = ["profile"], - ) - - strip_debug_symbols_feature = feature( - name = "strip_debug_symbols", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-Wl,-S"], - expand_if_available = "strip_debug_symbols", - ), - ], - ), - ], - ) - - build_interface_libraries_feature = feature( - name = "build_interface_libraries", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = [ - "%{generate_interface_library}", - "%{interface_library_builder_path}", - "%{interface_library_input_path}", - "%{interface_library_output_path}", - ], - expand_if_available = "generate_interface_library", - ), - ], - with_features = [ - with_feature_set( - features = ["supports_interface_shared_libraries"], - ), - ], - ), - ], - ) - - libraries_to_link_feature = feature( - name = "libraries_to_link", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - iterate_over = "libraries_to_link", - flag_groups = [ - flag_group( - flags = ["-Wl,--start-lib"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - flag_group( - flags = ["-Wl,-whole-archive"], - expand_if_true = - "libraries_to_link.is_whole_archive", - ), - flag_group( - flags = ["%{libraries_to_link.object_files}"], - iterate_over = "libraries_to_link.object_files", - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "interface_library", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "static_library", - ), - ), - flag_group( - flags = ["-l%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "dynamic_library", - ), - ), - flag_group( - flags = ["-l:%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "versioned_dynamic_library", - ), - ), - flag_group( - flags = ["-Wl,-no-whole-archive"], - expand_if_true = "libraries_to_link.is_whole_archive", - ), - flag_group( - flags = ["-Wl,--end-lib"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - ], - expand_if_available = "libraries_to_link", - ), - flag_group( - flags = ["-Wl,@%{thinlto_param_file}"], - expand_if_true = "thinlto_param_file", - ), - ], - ), - ], - ) - - user_link_flags_feature = feature( - name = "user_link_flags", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["%{user_link_flags}"], - iterate_over = "user_link_flags", - expand_if_available = "user_link_flags", - ), - ] + ([flag_group(flags = ctx.attr.link_libs)] if ctx.attr.link_libs else []), - ), - ], - ) - - fdo_prefetch_hints_feature = feature( - name = "fdo_prefetch_hints", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.lto_backend, - ], - flag_groups = [ - flag_group( - flags = [ - "-Xclang-only=-mllvm", - "-Xclang-only=-prefetch-hints-file=%{fdo_prefetch_hints_path}", - ], - expand_if_available = "fdo_prefetch_hints_path", - ), - ], - ), - ], - ) - - linkstamps_feature = feature( - name = "linkstamps", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["%{linkstamp_paths}"], - iterate_over = "linkstamp_paths", - expand_if_available = "linkstamp_paths", - ), - ], - ), - ], - ) - - gcc_coverage_map_format_feature = feature( - name = "gcc_coverage_map_format", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - "objc-executable", - "objc++-executable", - ], - flag_groups = [ - flag_group( - flags = ["-fprofile-arcs", "-ftest-coverage"], - expand_if_available = "gcov_gcno_file", - ), - ], - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [flag_group(flags = ["--coverage"])], - ), - ], - requires = [feature_set(features = ["coverage"])], - provides = ["profile"], - ) - - archiver_flags_feature = feature( - name = "archiver_flags", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.cpp_link_static_library], - flag_groups = [ - flag_group(flags = ["rcsD"]), - flag_group( - flags = ["%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - flag_set( - actions = [ACTION_NAMES.cpp_link_static_library], - flag_groups = [ - flag_group( - iterate_over = "libraries_to_link", - flag_groups = [ - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file", - ), - ), - flag_group( - flags = ["%{libraries_to_link.object_files}"], - iterate_over = "libraries_to_link.object_files", - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - ], - expand_if_available = "libraries_to_link", - ), - ], - ), - ], - ) - - force_pic_flags_feature = feature( - name = "force_pic_flags", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.lto_index_for_executable, - ], - flag_groups = [ - flag_group( - flags = ["-pie"], - expand_if_available = "force_pic", - ), - ], - ), - ], - ) - - dependency_file_feature = feature( - name = "dependency_file", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["-MD", "-MF", "%{dependency_file}"], - expand_if_available = "dependency_file", - ), - ], - ), - ], - ) - - dynamic_library_linker_tool_path = tool_paths - dynamic_library_linker_tool_feature = feature( - name = "dynamic_library_linker_tool", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = [" + cppLinkDynamicLibraryToolPath + "], - expand_if_available = "generate_interface_library", - ), - ], - with_features = [ - with_feature_set( - features = ["supports_interface_shared_libraries"], - ), - ], - ), - ], - ) - - output_execpath_flags_feature = feature( - name = "output_execpath_flags", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-o", "%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - ], - ) - - # Note that we also set --coverage for c++-link-nodeps-dynamic-library. The - # generated code contains references to gcov symbols, and the dynamic linker - # can't resolve them unless the library is linked against gcov. - coverage_feature = feature( - name = "coverage", - provides = ["profile"], - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = ([ - flag_group(flags = ctx.attr.coverage_compile_flags), - ] if ctx.attr.coverage_compile_flags else []), - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = ([ - flag_group(flags = ctx.attr.coverage_link_flags), - ] if ctx.attr.coverage_link_flags else []), - ), - ], - ) - - is_linux = ctx.attr.target_libc != "macosx" - - # TODO(#8303): Mac crosstool should also declare every feature. - if is_linux: - features = [ - dependency_file_feature, - random_seed_feature, - pic_feature, - per_object_debug_info_feature, - preprocessor_defines_feature, - includes_feature, - include_paths_feature, - fdo_instrument_feature, - cs_fdo_instrument_feature, - cs_fdo_optimize_feature, - fdo_prefetch_hints_feature, - autofdo_feature, - build_interface_libraries_feature, - dynamic_library_linker_tool_feature, - symbol_counts_feature, - shared_flag_feature, - linkstamps_feature, - output_execpath_flags_feature, - runtime_library_search_directories_feature, - library_search_directories_feature, - archiver_flags_feature, - force_pic_flags_feature, - fission_support_feature, - strip_debug_symbols_feature, - coverage_feature, - supports_pic_feature, - ] + ( - [ - supports_start_end_lib_feature, - ] if ctx.attr.supports_start_end_lib else [] - ) + [ - default_compile_flags_feature, - default_link_flags_feature, - libraries_to_link_feature, - user_link_flags_feature, - static_libgcc_feature, - fdo_optimize_feature, - supports_dynamic_linker_feature, - dbg_feature, - opt_feature, - user_compile_flags_feature, - sysroot_feature, - unfiltered_compile_flags_feature, - ] - else: - features = [ - supports_pic_feature, - ] + ( - [ - supports_start_end_lib_feature, - ] if ctx.attr.supports_start_end_lib else [] - ) + [ - coverage_feature, - default_compile_flags_feature, - default_link_flags_feature, - fdo_optimize_feature, - supports_dynamic_linker_feature, - dbg_feature, - opt_feature, - user_compile_flags_feature, - sysroot_feature, - unfiltered_compile_flags_feature, - ] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - cxx_builtin_include_directories = ctx.attr.cxx_builtin_include_directories, - toolchain_identifier = ctx.attr.toolchain_identifier, - host_system_name = ctx.attr.host_system_name, - target_system_name = ctx.attr.target_system_name, - target_cpu = ctx.attr.cpu, - target_libc = ctx.attr.target_libc, - compiler = ctx.attr.compiler, - abi_version = ctx.attr.abi_version, - abi_libc_version = ctx.attr.abi_libc_version, - tool_paths = tool_paths, - ) - -cc_toolchain_config = rule( - implementation = _impl, - attrs = { - "cpu": attr.string(mandatory = True), - "compiler": attr.string(mandatory = True), - "toolchain_identifier": attr.string(mandatory = True), - "host_system_name": attr.string(mandatory = True), - "target_system_name": attr.string(mandatory = True), - "target_libc": attr.string(mandatory = True), - "abi_version": attr.string(mandatory = True), - "abi_libc_version": attr.string(mandatory = True), - "cxx_builtin_include_directories": attr.string_list(), - "tool_paths": attr.string_dict(), - "compile_flags": attr.string_list(), - "dbg_compile_flags": attr.string_list(), - "opt_compile_flags": attr.string_list(), - "cxx_flags": attr.string_list(), - "link_flags": attr.string_list(), - "link_libs": attr.string_list(), - "opt_link_flags": attr.string_list(), - "unfiltered_compile_flags": attr.string_list(), - "coverage_compile_flags": attr.string_list(), - "coverage_link_flags": attr.string_list(), - "supports_start_end_lib": attr.bool(), - }, - provides = [CcToolchainConfigInfo], -) diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_wrapper.sh b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_wrapper.sh deleted file mode 100755 index b7ff6355883c..000000000000 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc/cc_wrapper.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Ship the environment to the C++ action -# -set -eu - -# Set-up the environment - - -# Call the C++ compiler -/usr/lib/llvm-8/bin/clang "$@" diff --git a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/config/BUILD b/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/config/BUILD deleted file mode 100644 index 3d87dd780c53..000000000000 --- a/bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/config/BUILD +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is auto-generated by an rbe_autoconfig repository rule -# and should not be modified directly. -# See @bazel_toolchains//rules:rbe_repo.bzl - -package(default_visibility = ["//visibility:public"]) - -toolchain( - name = "cc-toolchain", - exec_compatible_with = [ - "@bazel_tools//platforms:x86_64", - "@bazel_tools//platforms:linux", - "@bazel_tools//tools/cpp:clang", - ], - target_compatible_with = [ - "@bazel_tools//platforms:linux", - "@bazel_tools//platforms:x86_64", - ], - toolchain = "//bazel/toolchains/configs/clang_libcxx/bazel_0.29.1/cc:cc-compiler-k8", - toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", -) - -platform( - name = "platform", - constraint_values = [ - "@bazel_tools//platforms:x86_64", - "@bazel_tools//platforms:linux", - "@bazel_tools//tools/cpp:clang", - ], - remote_execution_properties = """ - properties: { - name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" - } - properties { - name: "OSFamily" - value: "Linux" - } - """, -) diff --git a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/BUILD b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/BUILD deleted file mode 100755 index ae7728d61bd4..000000000000 --- a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/BUILD +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes. - -package(default_visibility = ["//visibility:public"]) - -load(":cc_toolchain_config.bzl", "cc_toolchain_config") -load(":armeabi_cc_toolchain_config.bzl", "armeabi_cc_toolchain_config") -load("@rules_cc//cc:defs.bzl", "cc_toolchain", "cc_toolchain_suite") - -licenses(["notice"]) # Apache 2.0 - -cc_library( - name = "malloc", -) - -filegroup( - name = "empty", - srcs = [], -) - -filegroup( - name = "cc_wrapper", - srcs = ["cc_wrapper.sh"], -) - -filegroup( - name = "compiler_deps", - srcs = glob(["extra_tools/**"], allow_empty = True) + [":builtin_include_directory_paths"], -) - -# This is the entry point for --crosstool_top. Toolchains are found -# by lopping off the name of --crosstool_top and searching for -# the "${CPU}" entry in the toolchains attribute. -cc_toolchain_suite( - name = "toolchain", - toolchains = { - "k8|gcc": ":cc-compiler-k8", - "k8": ":cc-compiler-k8", - "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a", - "armeabi-v7a": ":cc-compiler-armeabi-v7a", - }, -) - -cc_toolchain( - name = "cc-compiler-k8", - toolchain_identifier = "local", - toolchain_config = ":local", - all_files = ":compiler_deps", - ar_files = ":compiler_deps", - as_files = ":compiler_deps", - compiler_files = ":compiler_deps", - dwp_files = ":empty", - linker_files = ":compiler_deps", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 1, -) - -cc_toolchain_config( - name = "local", - cpu = "k8", - compiler = "gcc", - toolchain_identifier = "local", - host_system_name = "local", - target_system_name = "local", - target_libc = "local", - abi_version = "local", - abi_libc_version = "local", - cxx_builtin_include_directories = ["/usr/lib/gcc/x86_64-linux-gnu/7/include", - "/usr/local/include", - "/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed", - "/usr/include/x86_64-linux-gnu", - "/usr/include", - "/usr/include/c++/7", - "/usr/include/x86_64-linux-gnu/c++/7", - "/usr/include/c++/7/backward"], - tool_paths = {"ar": "/usr/bin/ar", - "ld": "/usr/bin/ld", - "cpp": "/usr/bin/cpp", - "gcc": "/usr/bin/gcc", - "dwp": "/usr/bin/dwp", - "gcov": "None", - "nm": "/usr/bin/nm", - "objcopy": "/usr/bin/objcopy", - "objdump": "/usr/bin/objdump", - "strip": "/usr/bin/strip"}, - compile_flags = ["-U_FORTIFY_SOURCE", - "-fstack-protector", - "-Wall", - "-Wunused-but-set-parameter", - "-Wno-free-nonheap-object", - "-fno-omit-frame-pointer"], - opt_compile_flags = ["-g0", - "-O2", - "-D_FORTIFY_SOURCE=1", - "-DNDEBUG", - "-ffunction-sections", - "-fdata-sections"], - dbg_compile_flags = ["-g"], - cxx_flags = ["-std=c++0x"], - link_flags = ["-fuse-ld=gold", - "-Wl,-no-as-needed", - "-Wl,-z,relro,-z,now", - "-B/usr/bin", - "-pass-exit-codes", - "-lm"], - link_libs = ["-l:libstdc++.a"], - opt_link_flags = ["-Wl,--gc-sections"], - unfiltered_compile_flags = ["-fno-canonical-system-headers", - "-Wno-builtin-macro-redefined", - "-D__DATE__=\"redacted\"", - "-D__TIMESTAMP__=\"redacted\"", - "-D__TIME__=\"redacted\""], - coverage_compile_flags = ["--coverage"], - coverage_link_flags = ["--coverage"], - supports_start_end_lib = True, -) - -# Android tooling requires a default toolchain for the armeabi-v7a cpu. -cc_toolchain( - name = "cc-compiler-armeabi-v7a", - toolchain_identifier = "stub_armeabi-v7a", - toolchain_config = ":stub_armeabi-v7a", - all_files = ":empty", - ar_files = ":empty", - as_files = ":empty", - compiler_files = ":empty", - dwp_files = ":empty", - linker_files = ":empty", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 1, -) - -armeabi_cc_toolchain_config(name = "stub_armeabi-v7a") diff --git a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl deleted file mode 100755 index 94e0720bf6c9..000000000000 --- a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/armeabi_cc_toolchain_config.bzl +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A Starlark cc_toolchain configuration rule""" - -load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "feature", - "tool_path", -) - -def _impl(ctx): - toolchain_identifier = "stub_armeabi-v7a" - host_system_name = "armeabi-v7a" - target_system_name = "armeabi-v7a" - target_cpu = "armeabi-v7a" - target_libc = "armeabi-v7a" - compiler = "compiler" - abi_version = "armeabi-v7a" - abi_libc_version = "armeabi-v7a" - cc_target_os = None - builtin_sysroot = None - action_configs = [] - - supports_pic_feature = feature(name = "supports_pic", enabled = True) - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - features = [supports_dynamic_linker_feature, supports_pic_feature] - - cxx_builtin_include_directories = [] - artifact_name_patterns = [] - make_variables = [] - - tool_paths = [ - tool_path(name = "ar", path = "/bin/false"), - tool_path(name = "compat-ld", path = "/bin/false"), - tool_path(name = "cpp", path = "/bin/false"), - tool_path(name = "dwp", path = "/bin/false"), - tool_path(name = "gcc", path = "/bin/false"), - tool_path(name = "gcov", path = "/bin/false"), - tool_path(name = "ld", path = "/bin/false"), - tool_path(name = "nm", path = "/bin/false"), - tool_path(name = "objcopy", path = "/bin/false"), - tool_path(name = "objdump", path = "/bin/false"), - tool_path(name = "strip", path = "/bin/false"), - ] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - artifact_name_patterns = artifact_name_patterns, - cxx_builtin_include_directories = cxx_builtin_include_directories, - toolchain_identifier = toolchain_identifier, - host_system_name = host_system_name, - target_system_name = target_system_name, - target_cpu = target_cpu, - target_libc = target_libc, - compiler = compiler, - abi_version = abi_version, - abi_libc_version = abi_libc_version, - tool_paths = tool_paths, - make_variables = make_variables, - builtin_sysroot = builtin_sysroot, - cc_target_os = cc_target_os, - ) - -armeabi_cc_toolchain_config = rule( - implementation = _impl, - attrs = {}, - provides = [CcToolchainConfigInfo], -) diff --git a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/builtin_include_directory_paths b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/builtin_include_directory_paths deleted file mode 100755 index 30a600ae8b06..000000000000 --- a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/builtin_include_directory_paths +++ /dev/null @@ -1,14 +0,0 @@ -This file is generated by cc_configure and contains builtin include directories -that /usr/bin/gcc reported. This file is a dependency of every compilation action and -changes to it will be reflected in the action cache key. When some of these -paths change, Bazel will make sure to rerun the action, even though none of -declared action inputs or the action commandline changes. - -/usr/lib/gcc/x86_64-linux-gnu/7/include -/usr/local/include -/usr/lib/gcc/x86_64-linux-gnu/7/include-fixed -/usr/include/x86_64-linux-gnu -/usr/include -/usr/include/c++/7 -/usr/include/x86_64-linux-gnu/c++/7 -/usr/include/c++/7/backward diff --git a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_toolchain_config.bzl b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_toolchain_config.bzl deleted file mode 100755 index bf4d83940f89..000000000000 --- a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_toolchain_config.bzl +++ /dev/null @@ -1,1133 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A Starlark cc_toolchain configuration rule""" - -load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "feature", - "feature_set", - "flag_group", - "flag_set", - "tool_path", - "variable_with_value", - "with_feature_set", -) -load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") - -all_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ACTION_NAMES.lto_backend, -] - -all_cpp_compile_actions = [ - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, -] - -preprocessor_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, -] - -codegen_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, -] - -all_link_actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, -] - -lto_index_actions = [ - ACTION_NAMES.lto_index_for_executable, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, -] - -def _impl(ctx): - tool_paths = [ - tool_path(name = name, path = path) - for name, path in ctx.attr.tool_paths.items() - ] - action_configs = [] - - supports_pic_feature = feature( - name = "supports_pic", - enabled = True, - ) - supports_start_end_lib_feature = feature( - name = "supports_start_end_lib", - enabled = True, - ) - - default_compile_flags_feature = feature( - name = "default_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.compile_flags, - ), - ] if ctx.attr.compile_flags else []), - ), - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.dbg_compile_flags, - ), - ] if ctx.attr.dbg_compile_flags else []), - with_features = [with_feature_set(features = ["dbg"])], - ), - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.opt_compile_flags, - ), - ] if ctx.attr.opt_compile_flags else []), - with_features = [with_feature_set(features = ["opt"])], - ), - flag_set( - actions = all_cpp_compile_actions + [ACTION_NAMES.lto_backend], - flag_groups = ([ - flag_group( - flags = ctx.attr.cxx_flags, - ), - ] if ctx.attr.cxx_flags else []), - ), - ], - ) - - default_link_flags_feature = feature( - name = "default_link_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.link_flags, - ), - ] if ctx.attr.link_flags else []), - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.opt_link_flags, - ), - ] if ctx.attr.opt_link_flags else []), - with_features = [with_feature_set(features = ["opt"])], - ), - ], - ) - - dbg_feature = feature(name = "dbg") - - opt_feature = feature(name = "opt") - - sysroot_feature = feature( - name = "sysroot", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ] + all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["--sysroot=%{sysroot}"], - expand_if_available = "sysroot", - ), - ], - ), - ], - ) - - fdo_optimize_feature = feature( - name = "fdo_optimize", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-use=%{fdo_profile_path}", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - - user_compile_flags_feature = feature( - name = "user_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = [ - flag_group( - flags = ["%{user_compile_flags}"], - iterate_over = "user_compile_flags", - expand_if_available = "user_compile_flags", - ), - ], - ), - ], - ) - - unfiltered_compile_flags_feature = feature( - name = "unfiltered_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_compile_actions, - flag_groups = ([ - flag_group( - flags = ctx.attr.unfiltered_compile_flags, - ), - ] if ctx.attr.unfiltered_compile_flags else []), - ), - ], - ) - - library_search_directories_feature = feature( - name = "library_search_directories", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-L%{library_search_directories}"], - iterate_over = "library_search_directories", - expand_if_available = "library_search_directories", - ), - ], - ), - ], - ) - - static_libgcc_feature = feature( - name = "static_libgcc", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.lto_index_for_executable, - ACTION_NAMES.lto_index_for_dynamic_library, - ], - flag_groups = [flag_group(flags = ["-static-libgcc"])], - with_features = [ - with_feature_set(features = ["static_link_cpp_runtimes"]), - ], - ), - ], - ) - - pic_feature = feature( - name = "pic", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = [ - flag_group(flags = ["-fPIC"], expand_if_available = "pic"), - ], - ), - ], - ) - - per_object_debug_info_feature = feature( - name = "per_object_debug_info", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_codegen, - ], - flag_groups = [ - flag_group( - flags = ["-gsplit-dwarf"], - expand_if_available = "per_object_debug_info_file", - ), - ], - ), - ], - ) - - preprocessor_defines_feature = feature( - name = "preprocessor_defines", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["-D%{preprocessor_defines}"], - iterate_over = "preprocessor_defines", - ), - ], - ), - ], - ) - - cs_fdo_optimize_feature = feature( - name = "cs_fdo_optimize", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.lto_backend], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-use=%{fdo_profile_path}", - "-Xclang-only=-Wno-profile-instr-unprofiled", - "-Xclang-only=-Wno-profile-instr-out-of-date", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["csprofile"], - ) - - autofdo_feature = feature( - name = "autofdo", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [ - flag_group( - flags = [ - "-fauto-profile=%{fdo_profile_path}", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - runtime_library_search_directories_feature = feature( - name = "runtime_library_search_directories", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - iterate_over = "runtime_library_search_directories", - flag_groups = [ - flag_group( - flags = [ - "-Wl,-rpath,$EXEC_ORIGIN/%{runtime_library_search_directories}", - ], - expand_if_true = "is_cc_test", - ), - flag_group( - flags = [ - "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", - ], - expand_if_false = "is_cc_test", - ), - ], - expand_if_available = - "runtime_library_search_directories", - ), - ], - with_features = [ - with_feature_set(features = ["static_link_cpp_runtimes"]), - ], - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - iterate_over = "runtime_library_search_directories", - flag_groups = [ - flag_group( - flags = [ - "-Wl,-rpath,$ORIGIN/%{runtime_library_search_directories}", - ], - ), - ], - expand_if_available = - "runtime_library_search_directories", - ), - ], - with_features = [ - with_feature_set( - not_features = ["static_link_cpp_runtimes"], - ), - ], - ), - ], - ) - - fission_support_feature = feature( - name = "fission_support", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-Wl,--gdb-index"], - expand_if_available = "is_using_fission", - ), - ], - ), - ], - ) - - shared_flag_feature = feature( - name = "shared_flag", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, - ], - flag_groups = [flag_group(flags = ["-shared"])], - ), - ], - ) - - random_seed_feature = feature( - name = "random_seed", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = [ - flag_group( - flags = ["-frandom-seed=%{output_file}"], - expand_if_available = "output_file", - ), - ], - ), - ], - ) - - includes_feature = feature( - name = "includes", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ], - flag_groups = [ - flag_group( - flags = ["-include", "%{includes}"], - iterate_over = "includes", - expand_if_available = "includes", - ), - ], - ), - ], - ) - - fdo_instrument_feature = feature( - name = "fdo_instrument", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ] + all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = [ - "-fprofile-generate=%{fdo_instrument_path}", - "-fno-data-sections", - ], - expand_if_available = "fdo_instrument_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - cs_fdo_instrument_feature = feature( - name = "cs_fdo_instrument", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.lto_backend, - ] + all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = [ - "-fcs-profile-generate=%{cs_fdo_instrument_path}", - ], - expand_if_available = "cs_fdo_instrument_path", - ), - ], - ), - ], - provides = ["csprofile"], - ) - - include_paths_feature = feature( - name = "include_paths", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ], - flag_groups = [ - flag_group( - flags = ["-iquote", "%{quote_include_paths}"], - iterate_over = "quote_include_paths", - ), - flag_group( - flags = ["-I%{include_paths}"], - iterate_over = "include_paths", - ), - flag_group( - flags = ["-isystem", "%{system_include_paths}"], - iterate_over = "system_include_paths", - ), - ], - ), - ], - ) - - symbol_counts_feature = feature( - name = "symbol_counts", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = [ - "-Wl,--print-symbol-counts=%{symbol_counts_output}", - ], - expand_if_available = "symbol_counts_output", - ), - ], - ), - ], - ) - - llvm_coverage_map_format_feature = feature( - name = "llvm_coverage_map_format", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-instr-generate", - "-fcoverage-mapping", - ], - ), - ], - ), - flag_set( - actions = all_link_actions + lto_index_actions + [ - "objc-executable", - "objc++-executable", - ], - flag_groups = [ - flag_group(flags = ["-fprofile-instr-generate"]), - ], - ), - ], - requires = [feature_set(features = ["coverage"])], - provides = ["profile"], - ) - - strip_debug_symbols_feature = feature( - name = "strip_debug_symbols", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-Wl,-S"], - expand_if_available = "strip_debug_symbols", - ), - ], - ), - ], - ) - - build_interface_libraries_feature = feature( - name = "build_interface_libraries", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = [ - "%{generate_interface_library}", - "%{interface_library_builder_path}", - "%{interface_library_input_path}", - "%{interface_library_output_path}", - ], - expand_if_available = "generate_interface_library", - ), - ], - with_features = [ - with_feature_set( - features = ["supports_interface_shared_libraries"], - ), - ], - ), - ], - ) - - libraries_to_link_feature = feature( - name = "libraries_to_link", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - iterate_over = "libraries_to_link", - flag_groups = [ - flag_group( - flags = ["-Wl,--start-lib"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - flag_group( - flags = ["-Wl,-whole-archive"], - expand_if_true = - "libraries_to_link.is_whole_archive", - ), - flag_group( - flags = ["%{libraries_to_link.object_files}"], - iterate_over = "libraries_to_link.object_files", - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "interface_library", - ), - ), - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "static_library", - ), - ), - flag_group( - flags = ["-l%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "dynamic_library", - ), - ), - flag_group( - flags = ["-l:%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "versioned_dynamic_library", - ), - ), - flag_group( - flags = ["-Wl,-no-whole-archive"], - expand_if_true = "libraries_to_link.is_whole_archive", - ), - flag_group( - flags = ["-Wl,--end-lib"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - ], - expand_if_available = "libraries_to_link", - ), - flag_group( - flags = ["-Wl,@%{thinlto_param_file}"], - expand_if_true = "thinlto_param_file", - ), - ], - ), - ], - ) - - user_link_flags_feature = feature( - name = "user_link_flags", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["%{user_link_flags}"], - iterate_over = "user_link_flags", - expand_if_available = "user_link_flags", - ), - ] + ([flag_group(flags = ctx.attr.link_libs)] if ctx.attr.link_libs else []), - ), - ], - ) - - fdo_prefetch_hints_feature = feature( - name = "fdo_prefetch_hints", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.lto_backend, - ], - flag_groups = [ - flag_group( - flags = [ - "-Xclang-only=-mllvm", - "-Xclang-only=-prefetch-hints-file=%{fdo_prefetch_hints_path}", - ], - expand_if_available = "fdo_prefetch_hints_path", - ), - ], - ), - ], - ) - - linkstamps_feature = feature( - name = "linkstamps", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["%{linkstamp_paths}"], - iterate_over = "linkstamp_paths", - expand_if_available = "linkstamp_paths", - ), - ], - ), - ], - ) - - gcc_coverage_map_format_feature = feature( - name = "gcc_coverage_map_format", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - "objc-executable", - "objc++-executable", - ], - flag_groups = [ - flag_group( - flags = ["-fprofile-arcs", "-ftest-coverage"], - expand_if_available = "gcov_gcno_file", - ), - ], - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [flag_group(flags = ["--coverage"])], - ), - ], - requires = [feature_set(features = ["coverage"])], - provides = ["profile"], - ) - - archiver_flags_feature = feature( - name = "archiver_flags", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.cpp_link_static_library], - flag_groups = [ - flag_group(flags = ["rcsD"]), - flag_group( - flags = ["%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - flag_set( - actions = [ACTION_NAMES.cpp_link_static_library], - flag_groups = [ - flag_group( - iterate_over = "libraries_to_link", - flag_groups = [ - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file", - ), - ), - flag_group( - flags = ["%{libraries_to_link.object_files}"], - iterate_over = "libraries_to_link.object_files", - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - ], - expand_if_available = "libraries_to_link", - ), - ], - ), - ], - ) - - force_pic_flags_feature = feature( - name = "force_pic_flags", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.lto_index_for_executable, - ], - flag_groups = [ - flag_group( - flags = ["-pie"], - expand_if_available = "force_pic", - ), - ], - ), - ], - ) - - dependency_file_feature = feature( - name = "dependency_file", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.objc_compile, - ACTION_NAMES.objcpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["-MD", "-MF", "%{dependency_file}"], - expand_if_available = "dependency_file", - ), - ], - ), - ], - ) - - dynamic_library_linker_tool_path = tool_paths - dynamic_library_linker_tool_feature = feature( - name = "dynamic_library_linker_tool", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.lto_index_for_dynamic_library, - ACTION_NAMES.lto_index_for_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = [" + cppLinkDynamicLibraryToolPath + "], - expand_if_available = "generate_interface_library", - ), - ], - with_features = [ - with_feature_set( - features = ["supports_interface_shared_libraries"], - ), - ], - ), - ], - ) - - output_execpath_flags_feature = feature( - name = "output_execpath_flags", - flag_sets = [ - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = [ - flag_group( - flags = ["-o", "%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - ], - ) - - # Note that we also set --coverage for c++-link-nodeps-dynamic-library. The - # generated code contains references to gcov symbols, and the dynamic linker - # can't resolve them unless the library is linked against gcov. - coverage_feature = feature( - name = "coverage", - provides = ["profile"], - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = ([ - flag_group(flags = ctx.attr.coverage_compile_flags), - ] if ctx.attr.coverage_compile_flags else []), - ), - flag_set( - actions = all_link_actions + lto_index_actions, - flag_groups = ([ - flag_group(flags = ctx.attr.coverage_link_flags), - ] if ctx.attr.coverage_link_flags else []), - ), - ], - ) - - is_linux = ctx.attr.target_libc != "macosx" - - # TODO(#8303): Mac crosstool should also declare every feature. - if is_linux: - features = [ - dependency_file_feature, - random_seed_feature, - pic_feature, - per_object_debug_info_feature, - preprocessor_defines_feature, - includes_feature, - include_paths_feature, - fdo_instrument_feature, - cs_fdo_instrument_feature, - cs_fdo_optimize_feature, - fdo_prefetch_hints_feature, - autofdo_feature, - build_interface_libraries_feature, - dynamic_library_linker_tool_feature, - symbol_counts_feature, - shared_flag_feature, - linkstamps_feature, - output_execpath_flags_feature, - runtime_library_search_directories_feature, - library_search_directories_feature, - archiver_flags_feature, - force_pic_flags_feature, - fission_support_feature, - strip_debug_symbols_feature, - coverage_feature, - supports_pic_feature, - ] + ( - [ - supports_start_end_lib_feature, - ] if ctx.attr.supports_start_end_lib else [] - ) + [ - default_compile_flags_feature, - default_link_flags_feature, - libraries_to_link_feature, - user_link_flags_feature, - static_libgcc_feature, - fdo_optimize_feature, - supports_dynamic_linker_feature, - dbg_feature, - opt_feature, - user_compile_flags_feature, - sysroot_feature, - unfiltered_compile_flags_feature, - ] - else: - features = [ - supports_pic_feature, - ] + ( - [ - supports_start_end_lib_feature, - ] if ctx.attr.supports_start_end_lib else [] - ) + [ - coverage_feature, - default_compile_flags_feature, - default_link_flags_feature, - fdo_optimize_feature, - supports_dynamic_linker_feature, - dbg_feature, - opt_feature, - user_compile_flags_feature, - sysroot_feature, - unfiltered_compile_flags_feature, - ] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - cxx_builtin_include_directories = ctx.attr.cxx_builtin_include_directories, - toolchain_identifier = ctx.attr.toolchain_identifier, - host_system_name = ctx.attr.host_system_name, - target_system_name = ctx.attr.target_system_name, - target_cpu = ctx.attr.cpu, - target_libc = ctx.attr.target_libc, - compiler = ctx.attr.compiler, - abi_version = ctx.attr.abi_version, - abi_libc_version = ctx.attr.abi_libc_version, - tool_paths = tool_paths, - ) - -cc_toolchain_config = rule( - implementation = _impl, - attrs = { - "cpu": attr.string(mandatory = True), - "compiler": attr.string(mandatory = True), - "toolchain_identifier": attr.string(mandatory = True), - "host_system_name": attr.string(mandatory = True), - "target_system_name": attr.string(mandatory = True), - "target_libc": attr.string(mandatory = True), - "abi_version": attr.string(mandatory = True), - "abi_libc_version": attr.string(mandatory = True), - "cxx_builtin_include_directories": attr.string_list(), - "tool_paths": attr.string_dict(), - "compile_flags": attr.string_list(), - "dbg_compile_flags": attr.string_list(), - "opt_compile_flags": attr.string_list(), - "cxx_flags": attr.string_list(), - "link_flags": attr.string_list(), - "link_libs": attr.string_list(), - "opt_link_flags": attr.string_list(), - "unfiltered_compile_flags": attr.string_list(), - "coverage_compile_flags": attr.string_list(), - "coverage_link_flags": attr.string_list(), - "supports_start_end_lib": attr.bool(), - }, - provides = [CcToolchainConfigInfo], -) diff --git a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_wrapper.sh b/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_wrapper.sh deleted file mode 100755 index f246528abf2e..000000000000 --- a/bazel/toolchains/configs/gcc/bazel_0.29.1/cc/cc_wrapper.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Ship the environment to the C++ action -# -set -eu - -# Set-up the environment - - -# Call the C++ compiler -/usr/bin/gcc "$@" diff --git a/bazel/toolchains/configs/gcc/bazel_0.29.1/config/BUILD b/bazel/toolchains/configs/gcc/bazel_0.29.1/config/BUILD deleted file mode 100644 index 6de35eeeaf23..000000000000 --- a/bazel/toolchains/configs/gcc/bazel_0.29.1/config/BUILD +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file is auto-generated by an rbe_autoconfig repository rule -# and should not be modified directly. -# See @bazel_toolchains//rules:rbe_repo.bzl - -package(default_visibility = ["//visibility:public"]) - -toolchain( - name = "cc-toolchain", - exec_compatible_with = [ - "@bazel_tools//platforms:x86_64", - "@bazel_tools//platforms:linux", - "@bazel_tools//tools/cpp:clang", - ], - target_compatible_with = [ - "@bazel_tools//platforms:linux", - "@bazel_tools//platforms:x86_64", - ], - toolchain = "//bazel/toolchains/configs/gcc/bazel_0.29.1/cc:cc-compiler-k8", - toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", -) - -platform( - name = "platform", - constraint_values = [ - "@bazel_tools//platforms:x86_64", - "@bazel_tools//platforms:linux", - "@bazel_tools//tools/cpp:clang", - ], - remote_execution_properties = """ - properties: { - name: "container-image" - value:"docker://gcr.io/envoy-ci/envoy-build@sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" - } - properties { - name: "OSFamily" - value: "Linux" - } - """, -) diff --git a/bazel/toolchains/configs/versions.bzl b/bazel/toolchains/configs/versions.bzl deleted file mode 100644 index bdf2a5c264ee..000000000000 --- a/bazel/toolchains/configs/versions.bzl +++ /dev/null @@ -1,18 +0,0 @@ -# Generated file, do not modify by hand -# Generated by 'rbe_ubuntu_gcc_gen' rbe_autoconfig rule -"""Definitions to be used in rbe_repo attr of an rbe_autoconf rule """ -toolchain_config_spec0 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "clang") -toolchain_config_spec1 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "clang", "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", "BAZEL_LINKOPTS": "-lm:-pthread:-fuse-ld=lld", "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", "GCOV": "llvm-profdata", "CC": "clang", "CXX": "clang++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", "BAZEL_CXXOPTS": "-stdlib=libc++", "CXXFLAGS": "-stdlib=libc++"}, java_home = None, name = "clang_libcxx") -toolchain_config_spec2 = struct(config_repos = [], create_cc_configs = True, create_java_configs = False, env = {"BAZEL_COMPILER": "gcc", "BAZEL_LINKLIBS": "-l%:libstdc++.a", "BAZEL_LINKOPTS": "-lm", "CC": "gcc", "CXX": "g++", "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin"}, java_home = None, name = "gcc") -_TOOLCHAIN_CONFIG_SPECS = [toolchain_config_spec0,toolchain_config_spec1,toolchain_config_spec2] -_BAZEL_TO_CONFIG_SPEC_NAMES = {"0.29.1": ["clang", "clang_libcxx", "gcc"]} -LATEST = "sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" -CONTAINER_TO_CONFIG_SPEC_NAMES = {"sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f": ["clang", "clang_libcxx", "gcc"]} -_DEFAULT_TOOLCHAIN_CONFIG_SPEC = toolchain_config_spec0 -TOOLCHAIN_CONFIG_AUTOGEN_SPEC = struct( - bazel_to_config_spec_names_map = _BAZEL_TO_CONFIG_SPEC_NAMES, - container_to_config_spec_names_map = CONTAINER_TO_CONFIG_SPEC_NAMES, - default_toolchain_config_spec = _DEFAULT_TOOLCHAIN_CONFIG_SPEC, - latest_container = LATEST, - toolchain_config_specs = _TOOLCHAIN_CONFIG_SPECS, - ) \ No newline at end of file diff --git a/bazel/toolchains/empty.bzl b/bazel/toolchains/empty.bzl deleted file mode 100644 index 3fc95e435327..000000000000 --- a/bazel/toolchains/empty.bzl +++ /dev/null @@ -1,18 +0,0 @@ -_BAZEL_TO_CONFIG_SPEC_NAMES = {} - -# sha256 digest of the latest version of the toolchain container. -LATEST = "" - -_CONTAINER_TO_CONFIG_SPEC_NAMES = {} - -_DEFAULT_TOOLCHAIN_CONFIG_SPEC = "" - -_TOOLCHAIN_CONFIG_SPECS = [] - -TOOLCHAIN_CONFIG_AUTOGEN_SPEC = struct( - bazel_to_config_spec_names_map = _BAZEL_TO_CONFIG_SPEC_NAMES, - container_to_config_spec_names_map = _CONTAINER_TO_CONFIG_SPEC_NAMES, - default_toolchain_config_spec = _DEFAULT_TOOLCHAIN_CONFIG_SPEC, - latest_container = LATEST, - toolchain_config_specs = _TOOLCHAIN_CONFIG_SPECS, -) diff --git a/bazel/toolchains/rbe_toolchains_config.bzl b/bazel/toolchains/rbe_toolchains_config.bzl deleted file mode 100644 index 8820710297a7..000000000000 --- a/bazel/toolchains/rbe_toolchains_config.bzl +++ /dev/null @@ -1,73 +0,0 @@ -load("@bazel_skylib//lib:dicts.bzl", "dicts") -load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig") -load("@envoy//bazel/toolchains:configs/versions.bzl", _generated_toolchain_config_suite_autogen_spec = "TOOLCHAIN_CONFIG_AUTOGEN_SPEC") - -_ENVOY_BUILD_IMAGE_REGISTRY = "gcr.io" -_ENVOY_BUILD_IMAGE_REPOSITORY = "envoy-ci/envoy-build" -_ENVOY_BUILD_IMAGE_DIGEST = "sha256:9236915d10004a35f2439ce4a1c33c1dbb06f95f84c4a4497d4e4f95cdc9e07f" -_CONFIGS_OUTPUT_BASE = "bazel/toolchains/configs" - -_CLANG_ENV = { - "BAZEL_COMPILER": "clang", - "BAZEL_LINKLIBS": "-l%:libstdc++.a", - "BAZEL_LINKOPTS": "-lm:-fuse-ld=lld", - "BAZEL_USE_LLVM_NATIVE_COVERAGE": "1", - "GCOV": "llvm-profdata", - "CC": "clang", - "CXX": "clang++", - "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", -} - -_CLANG_LIBCXX_ENV = dicts.add(_CLANG_ENV, { - "BAZEL_LINKLIBS": "-l%:libc++.a:-l%:libc++abi.a", - "BAZEL_LINKOPTS": "-lm:-pthread:-fuse-ld=lld", - "BAZEL_CXXOPTS": "-stdlib=libc++", - "CXXFLAGS": "-stdlib=libc++", -}) - -_GCC_ENV = { - "BAZEL_COMPILER": "gcc", - "BAZEL_LINKLIBS": "-l%:libstdc++.a", - "BAZEL_LINKOPTS": "-lm", - "CC": "gcc", - "CXX": "g++", - "PATH": "/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/llvm-8/bin", -} - -_TOOLCHAIN_CONFIG_SUITE_SPEC = { - "container_registry": _ENVOY_BUILD_IMAGE_REGISTRY, - "container_repo": _ENVOY_BUILD_IMAGE_REPOSITORY, - "output_base": _CONFIGS_OUTPUT_BASE, - "repo_name": "envoy", - "toolchain_config_suite_autogen_spec": _generated_toolchain_config_suite_autogen_spec, -} - -def _envoy_rbe_toolchain(name, env, toolchain_config_spec_name): - rbe_autoconfig( - name = name + "_gen", - export_configs = True, - create_java_configs = False, - digest = _ENVOY_BUILD_IMAGE_DIGEST, - registry = _ENVOY_BUILD_IMAGE_REGISTRY, - repository = _ENVOY_BUILD_IMAGE_REPOSITORY, - env = env, - toolchain_config_spec_name = toolchain_config_spec_name, - toolchain_config_suite_spec = _TOOLCHAIN_CONFIG_SUITE_SPEC, - use_checked_in_confs = "False", - ) - - rbe_autoconfig( - name = name, - create_java_configs = False, - digest = _ENVOY_BUILD_IMAGE_DIGEST, - registry = _ENVOY_BUILD_IMAGE_REGISTRY, - repository = _ENVOY_BUILD_IMAGE_REPOSITORY, - toolchain_config_spec_name = toolchain_config_spec_name, - toolchain_config_suite_spec = _TOOLCHAIN_CONFIG_SUITE_SPEC, - use_checked_in_confs = "Force", - ) - -def rbe_toolchains_config(): - _envoy_rbe_toolchain("rbe_ubuntu_clang", _CLANG_ENV, "clang") - _envoy_rbe_toolchain("rbe_ubuntu_clang_libcxx", _CLANG_LIBCXX_ENV, "clang_libcxx") - _envoy_rbe_toolchain("rbe_ubuntu_gcc", _GCC_ENV, "gcc") diff --git a/bazel/toolchains/regenerate.sh b/bazel/toolchains/regenerate.sh deleted file mode 100755 index b391a9efad9c..000000000000 --- a/bazel/toolchains/regenerate.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -e - -export RBE_AUTOCONF_ROOT=$(bazel info workspace) - -rm -rf "${RBE_AUTOCONF_ROOT}"/bazel/toolchains/configs/* -cp -vf "${RBE_AUTOCONF_ROOT}/bazel/toolchains/empty.bzl" "${RBE_AUTOCONF_ROOT}/bazel/toolchains/configs/versions.bzl" - -# Bazel query is the right command so bazel won't fail itself. -bazel query ${BAZEL_QUERY_OPTIONS} "@rbe_ubuntu_clang_gen//..." -bazel query ${BAZEL_QUERY_OPTIONS} "@rbe_ubuntu_clang_libcxx_gen//..." -bazel query ${BAZEL_QUERY_OPTIONS} "@rbe_ubuntu_gcc_gen//..." diff --git a/ci/build_container/Dockerfile-centos b/ci/build_container/Dockerfile-centos deleted file mode 100644 index bbfd259e2990..000000000000 --- a/ci/build_container/Dockerfile-centos +++ /dev/null @@ -1,7 +0,0 @@ -FROM centos:7 - -COPY ./build_container_common.sh / -COPY ./build_container_centos.sh / - -ENV PATH /opt/rh/rh-git218/root/usr/bin:/opt/rh/devtoolset-7/root/usr/bin:/opt/llvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -RUN ./build_container_centos.sh diff --git a/ci/build_container/Dockerfile-ubuntu b/ci/build_container/Dockerfile-ubuntu deleted file mode 100644 index 1b2586ca2a14..000000000000 --- a/ci/build_container/Dockerfile-ubuntu +++ /dev/null @@ -1,6 +0,0 @@ -FROM ubuntu:xenial - -COPY ./build_container_common.sh / -COPY ./build_container_ubuntu.sh / - -RUN ./build_container_ubuntu.sh diff --git a/ci/build_container/README.md b/ci/build_container/README.md deleted file mode 100644 index 4312a7005618..000000000000 --- a/ci/build_container/README.md +++ /dev/null @@ -1,8 +0,0 @@ -Envoy's CI has a build run called `build_image`. On a commit to master, `ci/build_container/docker_push.sh` -checks if the commit has changed the `ci/build_container` directory. If there are changes, CI builds a new `envoyproxy/envoy-build` -image. The image is pushed to [dockerhub](https://hub.docker.com/r/envoyproxy/envoy-build/tags/) under `latest` and under the commit sha. - -After the PR that changes `ci/build_container` has been merged, and the new image gets pushed, -a second PR is needed to update `ci/envoy_build_sha.sh`. In order to pull the new tagged version of -the build image, change ENVOY_BUILD_SHA [here](https://github.com/envoyproxy/envoy/blob/master/ci/envoy_build_sha.sh). -Any PRs that depend on this image change will have to merge master after the change to `ci/envoy_build_sha.sh` has been merged to master. diff --git a/ci/build_container/build_container_centos.sh b/ci/build_container/build_container_centos.sh deleted file mode 100755 index b1d15d166db0..000000000000 --- a/ci/build_container/build_container_centos.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -e - -# Note: rh-git218 is needed to run `git -C` in docs build process. -yum install -y centos-release-scl epel-release -yum update -y -yum install -y devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-binutils java-1.8.0-openjdk-headless rsync \ - rh-git218 wget unzip which make cmake3 patch ninja-build devtoolset-7-libatomic-devel openssl python27 \ - libtool autoconf tcpdump - -ln -s /usr/bin/cmake3 /usr/bin/cmake -ln -s /usr/bin/ninja-build /usr/bin/ninja - -# SLES 11 has older glibc than CentOS 7, so pre-built binary for it works on CentOS 7 -LLVM_VERSION=8.0.0 -LLVM_RELEASE="clang+llvm-${LLVM_VERSION}-x86_64-linux-sles11.3" -curl -OL "https://releases.llvm.org/${LLVM_VERSION}/${LLVM_RELEASE}.tar.xz" -tar Jxf "${LLVM_RELEASE}.tar.xz" -mv "./${LLVM_RELEASE}" /opt/llvm -rm "./${LLVM_RELEASE}.tar.xz" - -# httpd24 is equired by rh-git218 -echo "/opt/rh/httpd24/root/usr/lib64" > /etc/ld.so.conf.d/httpd24.conf -echo "/opt/llvm/lib" > /etc/ld.so.conf.d/llvm.conf -ldconfig - -# Setup tcpdump for non-root. -groupadd pcap -chgrp pcap /usr/sbin/tcpdump -chmod 750 /usr/sbin/tcpdump -setcap cap_net_raw,cap_net_admin=eip /usr/sbin/tcpdump - -./build_container_common.sh diff --git a/ci/build_container/build_container_common.sh b/ci/build_container/build_container_common.sh deleted file mode 100755 index 97bb0c2268de..000000000000 --- a/ci/build_container/build_container_common.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -e - -if [[ "$(uname -m)" == "x86_64" ]]; then - # buildifier - VERSION=0.28.0 - SHA256=3d474be62f8e18190546881daf3c6337d857bf371faf23f508e9b456b0244267 - curl --location --output /usr/local/bin/buildifier https://github.com/bazelbuild/buildtools/releases/download/"$VERSION"/buildifier \ - && echo "$SHA256 /usr/local/bin/buildifier" | sha256sum --check \ - && chmod +x /usr/local/bin/buildifier - - # bazelisk - VERSION=1.0 - SHA256=820f1432bb729cf1d51697a64ce57c0cff7ea4013acaf871b8c24b6388174d0d - curl --location --output /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v${VERSION}/bazelisk-linux-amd64 \ - && echo "$SHA256 /usr/local/bin/bazel" | sha256sum --check \ - && chmod +x /usr/local/bin/bazel -fi diff --git a/ci/build_container/build_container_ubuntu.sh b/ci/build_container/build_container_ubuntu.sh deleted file mode 100755 index e7ea9f625425..000000000000 --- a/ci/build_container/build_container_ubuntu.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -set -e - -ARCH="$(uname -m)" - -# Setup basic requirements and install them. -apt-get update -export DEBIAN_FRONTEND=noninteractive -apt-get install -y --no-install-recommends software-properties-common apt-transport-https - -# gcc-7 -add-apt-repository -y ppa:ubuntu-toolchain-r/test -apt-get update -apt-get install -y --no-install-recommends g++-7 -update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-7 1000 -update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 1000 -update-alternatives --config gcc -update-alternatives --config g++ - -apt-get install -y --no-install-recommends curl wget make cmake git python python-pip python-setuptools python3 python3-pip \ - unzip bc libtool ninja-build automake zip time gdb strace tshark tcpdump patch xz-utils rsync ssh-client - -# clang 8. -case $ARCH in - 'ppc64le' ) - LLVM_VERSION=8.0.0 - LLVM_RELEASE="clang+llvm-${LLVM_VERSION}-powerpc64le-unknown-unknown" - wget "https://releases.llvm.org/${LLVM_VERSION}/${LLVM_RELEASE}.tar.xz" - tar Jxf "${LLVM_RELEASE}.tar.xz" - mv "./${LLVM_RELEASE}" /opt/llvm - rm "./${LLVM_RELEASE}.tar.xz" - echo "/opt/llvm/lib" > /etc/ld.so.conf.d/llvm.conf - ldconfig - ;; - 'x86_64' ) - wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - - apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-8 main" - apt-get update - apt-get install -y --no-install-recommends clang-8 clang-format-8 clang-tidy-8 lld-8 libc++-8-dev libc++abi-8-dev llvm-8 - ;; -esac - -# Bazel and related dependencies. -case $ARCH in - 'ppc64le' ) - BAZEL_LATEST="$(curl https://oplab9.parqtec.unicamp.br/pub/ppc64el/bazel/ubuntu_16.04/latest/ 2>&1 \ - | sed -n 's/.*href="\([^"]*\).*/\1/p' | grep '^bazel' | head -n 1)" - curl -fSL https://oplab9.parqtec.unicamp.br/pub/ppc64el/bazel/ubuntu_16.04/latest/${BAZEL_LATEST} \ - -o /usr/local/bin/bazel - chmod +x /usr/local/bin/bazel - ;; -esac - -apt-get install -y aspell -rm -rf /var/lib/apt/lists/* - -# Setup tcpdump for non-root. -groupadd pcap -chgrp pcap /usr/sbin/tcpdump -chmod 750 /usr/sbin/tcpdump -setcap cap_net_raw,cap_net_admin=eip /usr/sbin/tcpdump - -# virtualenv -pip3 install virtualenv - -./build_container_common.sh - -apt-get clean diff --git a/ci/build_container/docker_build.sh b/ci/build_container/docker_build.sh deleted file mode 100755 index eb539b7c4f0d..000000000000 --- a/ci/build_container/docker_build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -[[ -z "${LINUX_DISTRO}" ]] && LINUX_DISTRO="ubuntu" -[[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME=envoyproxy/envoy-build-"${LINUX_DISTRO}" - -docker build -f Dockerfile-${LINUX_DISTRO} -t ${IMAGE_NAME}:$CIRCLE_SHA1 . diff --git a/ci/build_container/docker_push.sh b/ci/build_container/docker_push.sh deleted file mode 100755 index 395adef8a754..000000000000 --- a/ci/build_container/docker_push.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Do not ever set -x here, it is a security hazard as it will place the credentials below in the -# CircleCI logs. -set -e - -# push the envoy image on merge to master -branch_want_push='false' -for branch in "master" -do - if [[ "$CIRCLE_BRANCH" == "$branch" ]]; then - branch_want_push='true' - fi -done - -if [[ -z "${CIRCLE_PR_NUMBER}" && "${branch_want_push}" == "true" ]]; then - diff_base="HEAD^" -else - git fetch https://github.com/envoyproxy/envoy.git master - diff_base="$(git merge-base HEAD FETCH_HEAD)" -fi - -diff_want_build='false' -if [[ ! -z $(git diff --name-only "${diff_base}..HEAD" ci/build_container/) ]]; then - echo "There are changes in the ci/build_container directory" - diff_want_build='true' -fi - -cd ci/build_container -if [ "$diff_want_build" == "true" ]; then - for distro in ubuntu centos - do - echo "Updating envoyproxy/envoy-build-${distro} image" - LINUX_DISTRO=$distro ./docker_build.sh - done -else - echo "The ci/build_container directory has not changed" - exit 0 -fi - -if [[ -z "${CIRCLE_PR_NUMBER}" && "${branch_want_push}" == "true" ]]; then - docker login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_PASSWORD" - - if [[ ! -z "${GCP_SERVICE_ACCOUNT_KEY}" ]]; then - echo ${GCP_SERVICE_ACCOUNT_KEY} | base64 --decode | gcloud auth activate-service-account --key-file=- - gcloud auth configure-docker - fi - - for distro in ubuntu centos - do - echo "Updating envoyproxy/envoy-build-${distro} image" - docker push envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" - docker tag envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" envoyproxy/envoy-build-"${distro}":latest - docker push envoyproxy/envoy-build-"${distro}":latest - - if [[ "$distro" == "ubuntu" ]] - then - echo "Updating envoyproxy/envoy-build image" - docker tag envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" envoyproxy/envoy-build:"$CIRCLE_SHA1" - docker push envoyproxy/envoy-build:"$CIRCLE_SHA1" - docker tag envoyproxy/envoy-build:"$CIRCLE_SHA1" envoyproxy/envoy-build:latest - docker push envoyproxy/envoy-build:latest - - echo "Updating gcr.io/envoy-ci/envoy-build image" - docker tag envoyproxy/envoy-build-"${distro}":"$CIRCLE_SHA1" gcr.io/envoy-ci/envoy-build:"$CIRCLE_SHA1" - docker push gcr.io/envoy-ci/envoy-build:"$CIRCLE_SHA1" - fi - done -else - echo 'Ignoring PR branch for docker push.' -fi diff --git a/ci/envoy_build_sha.sh b/ci/envoy_build_sha.sh index bdc6fefe409d..5783a2a637bf 100644 --- a/ci/envoy_build_sha.sh +++ b/ci/envoy_build_sha.sh @@ -1,2 +1,2 @@ -ENVOY_BUILD_SHA=$(grep envoyproxy/envoy-build $(dirname $0)/../.circleci/config.yml | sed -e 's#.*envoyproxy/envoy-build:\(.*\)#\1#' | uniq) +ENVOY_BUILD_SHA=$(grep envoyproxy/envoy-build-ubuntu@sha256 $(dirname $0)/../.circleci/config.yml | sed -e 's#.*envoyproxy/envoy-build-ubuntu@sha256:\(.*\)#\1#' | uniq) [[ $(wc -l <<< "${ENVOY_BUILD_SHA}" | awk '{$1=$1};1') == 1 ]] || (echo ".circleci/config.yml hashes are inconsistent!" && exit 1) diff --git a/ci/run_clang_tidy.sh b/ci/run_clang_tidy.sh index 9ab612b48d03..75e7146727db 100755 --- a/ci/run_clang_tidy.sh +++ b/ci/run_clang_tidy.sh @@ -3,7 +3,7 @@ set -e # Quick syntax check of .clang-tidy using PyYAML. -if ! python -c 'import yaml, sys; yaml.safe_load(sys.stdin)' < .clang-tidy > /dev/null; then +if ! python3 -c 'import yaml, sys; yaml.safe_load(sys.stdin)' < .clang-tidy > /dev/null; then echo ".clang-tidy has a syntax error" exit 1 fi diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh index 89f2cb6db41b..5cf5acb2ef75 100755 --- a/ci/run_envoy_docker.sh +++ b/ci/run_envoy_docker.sh @@ -10,7 +10,7 @@ set -e USER=root USER_GROUP=root -[[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-ubuntu" +[[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-ubuntu@sha256" # The IMAGE_ID defaults to the CI hash but can be set to an arbitrary image ID (found with 'docker # images'). [[ -z "${IMAGE_ID}" ]] && IMAGE_ID="${ENVOY_BUILD_SHA}" From d6964e5aa985663f50759ad4cfc7f3d78d944b39 Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Thu, 26 Sep 2019 14:03:13 -0700 Subject: [PATCH 06/32] clarify xDS DiscoveryRequest behavior (#8334) Clarifies comments in discovery.proto about when response_nonce is(n't) set for (Delta)DiscoveryRequest. Also left a TODO in grpc_mux_impl_test.cc where spec-deviating behavior is expected. Risk Level: none Testing: comment-only change Signed-off-by: Fred Douglas --- api/envoy/api/v2/discovery.proto | 7 ++++--- api/envoy/api/v3alpha/discovery.proto | 7 ++++--- test/common/config/grpc_mux_impl_test.cc | 3 +++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index 2d04a24817f9..a8423f5f904f 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -44,8 +44,9 @@ message DiscoveryRequest { // nonce corresponding to DiscoveryResponse being ACK/NACKed. See above // discussion on version_info and the DiscoveryResponse nonce comment. This - // may be empty if no nonce is available, e.g. at startup or for non-stream - // xDS implementations. + // may be empty only if 1) this is a non-persistent-stream xDS such as HTTP, + // or 2) the client has not yet accepted an update in this xDS stream (unlike + // delta, where it is populated only for new explicit ACKs). string response_nonce = 5; // This is populated when the previous :ref:`DiscoveryResponse ` @@ -176,7 +177,7 @@ message DeltaDiscoveryRequest { // When the DeltaDiscoveryRequest is a ACK or NACK message in response // to a previous DeltaDiscoveryResponse, the response_nonce must be the // nonce in the DeltaDiscoveryResponse. - // Otherwise response_nonce must be omitted. + // Otherwise (unlike in DiscoveryRequest) response_nonce must be omitted. string response_nonce = 6; // This is populated when the previous :ref:`DiscoveryResponse ` diff --git a/api/envoy/api/v3alpha/discovery.proto b/api/envoy/api/v3alpha/discovery.proto index 1547b030062f..c03bb57cebd5 100644 --- a/api/envoy/api/v3alpha/discovery.proto +++ b/api/envoy/api/v3alpha/discovery.proto @@ -44,8 +44,9 @@ message DiscoveryRequest { // nonce corresponding to DiscoveryResponse being ACK/NACKed. See above // discussion on version_info and the DiscoveryResponse nonce comment. This - // may be empty if no nonce is available, e.g. at startup or for non-stream - // xDS implementations. + // may be empty only if 1) this is a non-persistent-stream xDS such as HTTP, + // or 2) the client has not yet accepted an update in this xDS stream (unlike + // delta, where it is populated only for new explicit ACKs). string response_nonce = 5; // This is populated when the previous :ref:`DiscoveryResponse ` @@ -176,7 +177,7 @@ message DeltaDiscoveryRequest { // When the DeltaDiscoveryRequest is a ACK or NACK message in response // to a previous DeltaDiscoveryResponse, the response_nonce must be the // nonce in the DeltaDiscoveryResponse. - // Otherwise response_nonce must be omitted. + // Otherwise (unlike in DiscoveryRequest) response_nonce must be omitted. string response_nonce = 6; // This is populated when the previous :ref:`DiscoveryResponse ` diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index c2493c0934f5..b266dda51b0c 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -580,6 +580,9 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { response->set_version_info("1"); response->set_type_url(type_url); + // TODO(fredlas) the expectation of no discovery request here is against the xDS spec. + // The upcoming xDS overhaul (part of/followup to PR7293) will fix this. + // // This contains zero resources. No discovery request should be sent. grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); From 6d772458b1250c9f067f008a1a3c3dcfc1ce2a15 Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Thu, 26 Sep 2019 15:19:09 -0600 Subject: [PATCH 07/32] use envoy_google_grpc_external_deps for google_grpc_context_lib (#8380) Directly use "external_deps" in google_grpc_context_lib causing a problem during Google import rewriting. Signed-off-by: Wayne Zhang --- source/common/grpc/BUILD | 4 ++-- source/common/grpc/google_grpc_context.cc | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index f4d6d906e70c..c140f561e31d 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -3,6 +3,7 @@ licenses(["notice"]) # Apache 2 load( "//bazel:envoy_build_system.bzl", "envoy_cc_library", + "envoy_google_grpc_external_deps", "envoy_package", "envoy_select_google_grpc", ) @@ -162,13 +163,12 @@ envoy_cc_library( name = "google_grpc_context_lib", srcs = ["google_grpc_context.cc"], hdrs = ["google_grpc_context.h"], - external_deps = ["grpc"], deps = [ "//source/common/common:assert_lib", "//source/common/common:lock_guard_lib", "//source/common/common:macros", "//source/common/common:thread_lib", - ], + ] + envoy_google_grpc_external_deps(), ) envoy_cc_library( diff --git a/source/common/grpc/google_grpc_context.cc b/source/common/grpc/google_grpc_context.cc index b2b27ce8065a..a8ad82979ec6 100644 --- a/source/common/grpc/google_grpc_context.cc +++ b/source/common/grpc/google_grpc_context.cc @@ -7,7 +7,9 @@ #include "common/common/macros.h" #include "common/common/thread.h" +#ifdef ENVOY_GOOGLE_GRPC #include "grpcpp/grpcpp.h" +#endif namespace Envoy { namespace Grpc { @@ -15,7 +17,9 @@ namespace Grpc { GoogleGrpcContext::GoogleGrpcContext() : instance_tracker_(instanceTracker()) { Thread::LockGuard lock(instance_tracker_.mutex_); if (++instance_tracker_.live_instances_ == 1) { +#ifdef ENVOY_GOOGLE_GRPC grpc_init(); +#endif } } @@ -28,7 +32,9 @@ GoogleGrpcContext::~GoogleGrpcContext() { Thread::LockGuard lock(instance_tracker_.mutex_); ASSERT(instance_tracker_.live_instances_ > 0); if (--instance_tracker_.live_instances_ == 0) { +#ifdef ENVOY_GOOGLE_GRPC grpc_shutdown_blocking(); // Waiting for quiescence avoids non-determinism in tests. +#endif } } From 60707bdc37a52514cce8465a7bba84774321eacd Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Thu, 26 Sep 2019 14:34:58 -0700 Subject: [PATCH 08/32] config: WatchMap: match old GrpcMuxImpl's state-of-the-world behavior (#8350) In bringing my draft of the delta+SotW xDS unification to a state of compiling+passing tests, I found that WatchMap's SotW handling was missing some special case logic that exists in the existing/old GrpcMuxImpl. Specifically, eliding onConfigUpdate when it would be an "update" from empty to empty, and rejecting a (non-empty) SotW update when the type URL has no watches whatsoever. Added the eliding, but we decided the rejection is unnecessary; WatchMap will still not do it. Risk Level: low Testing: updated watch_map_test Signed-off-by: Fred Douglas --- source/common/config/watch_map.cc | 18 +++++--- source/common/config/watch_map.h | 4 ++ test/common/config/watch_map_test.cc | 65 ++++++++++++++++++++-------- tools/spelling_dictionary.txt | 1 + 4 files changed, 66 insertions(+), 22 deletions(-) diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index f351dd962a2f..adc99f145f55 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -54,7 +54,6 @@ absl::flat_hash_set WatchMap::watchesInterestedIn(const std::string& res void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { if (watches_.empty()) { - ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); return; } SubscriptionCallbacks& name_getter = (*watches_.begin())->callbacks_; @@ -71,16 +70,25 @@ void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField } } + const bool map_is_single_wildcard = (watches_.size() == 1 && wildcard_watches_.size() == 1); // We just bundled up the updates into nice per-watch packages. Now, deliver them. for (auto& watch : watches_) { const auto this_watch_updates = per_watch_updates.find(watch); if (this_watch_updates == per_watch_updates.end()) { - // This update included no resources this watch cares about - so we do an empty - // onConfigUpdate(), to notify the watch that its resources - if they existed before this - - // were dropped. - watch->callbacks_.onConfigUpdate({}, version_info); + // This update included no resources this watch cares about. + // 1) If there is only a single, wildcard watch (i.e. Cluster or Listener), always call + // its onConfigUpdate even if just a no-op, to properly maintain state-of-the-world + // semantics and the update_empty stat. + // 2) If this watch previously had some resources, it means this update is removing all + // of this watch's resources, so the watch must be informed with an onConfigUpdate. + // 3) Otherwise, we can skip onConfigUpdate for this watch. + if (map_is_single_wildcard || !watch->state_of_the_world_empty_) { + watch->callbacks_.onConfigUpdate({}, version_info); + watch->state_of_the_world_empty_ = true; + } } else { watch->callbacks_.onConfigUpdate(this_watch_updates->second, version_info); + watch->state_of_the_world_empty_ = false; } } } diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index 5e75e5e88dd7..9df852c7f712 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -26,6 +26,10 @@ struct Watch { Watch(SubscriptionCallbacks& callbacks) : callbacks_(callbacks) {} SubscriptionCallbacks& callbacks_; std::set resource_names_; // must be sorted set, for set_difference. + // Needed only for state-of-the-world. + // Whether the most recent update contained any resources this watch cares about. + // If true, a new update that also contains no resources can skip this watch. + bool state_of_the_world_empty_{true}; }; // NOTE: Users are responsible for eventually calling removeWatch() on the Watch* returned diff --git a/test/common/config/watch_map_test.cc b/test/common/config/watch_map_test.cc index 543298557fab..77003b2a61c5 100644 --- a/test/common/config/watch_map_test.cc +++ b/test/common/config/watch_map_test.cc @@ -67,13 +67,17 @@ void expectDeltaAndSotwUpdate( })); } -// Sometimes we want to verify that a delta onConfigUpdate simply doesn't happen. However, for SotW, -// every update triggers all onConfigUpdate()s, so we should still expect empty calls for that. -void expectNoDeltaUpdate(NamedMockSubscriptionCallbacks& callbacks, const std::string& version) { +void expectNoUpdate(NamedMockSubscriptionCallbacks& callbacks, const std::string& version) { + EXPECT_CALL(callbacks, onConfigUpdate(_, version)).Times(0); + EXPECT_CALL(callbacks, onConfigUpdate(_, _, version)).Times(0); +} + +void expectEmptySotwNoDeltaUpdate(NamedMockSubscriptionCallbacks& callbacks, + const std::string& version) { EXPECT_CALL(callbacks, onConfigUpdate(_, version)) .WillOnce(Invoke([](const Protobuf::RepeatedPtrField& gotten_resources, - const std::string&) { EXPECT_EQ(0, gotten_resources.size()); })); - EXPECT_CALL(callbacks, onConfigUpdate(_, _, _)).Times(0); + const std::string&) { EXPECT_EQ(gotten_resources.size(), 0); })); + EXPECT_CALL(callbacks, onConfigUpdate(_, _, version)).Times(0); } Protobuf::RepeatedPtrField @@ -105,7 +109,7 @@ void doDeltaAndSotwUpdate(SubscriptionCallbacks& watch_map, for (const auto& n : removed_names) { *removed_names_proto.Add() = n; } - watch_map.onConfigUpdate(delta_resources, removed_names_proto, "version1"); + watch_map.onConfigUpdate(delta_resources, removed_names_proto, version); } // Tests the simple case of a single watch. Checks that the watch will not be told of updates to @@ -195,9 +199,9 @@ TEST(WatchMapTest, Overlap) { EXPECT_TRUE(added_removed.removed_.empty()); watch_map.updateWatchInterest(watch2, {"dummy"}); - // First watch receives update. + // *Only* first watch receives update. expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version1"); - expectNoDeltaUpdate(callbacks2, "version1"); + expectNoUpdate(callbacks2, "version1"); doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); } // Second watch becomes interested. @@ -218,9 +222,13 @@ TEST(WatchMapTest, Overlap) { EXPECT_TRUE(added_removed.added_.empty()); // nothing happens EXPECT_TRUE(added_removed.removed_.empty()); - // *Only* second watch receives update. - expectNoDeltaUpdate(callbacks1, "version3"); + // Both watches receive the update. For watch2, this is obviously desired. expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version3"); + // For watch1, it's more subtle: the WatchMap sees that this update has no + // resources watch1 cares about, but also knows that watch1 previously had + // some resources. So, it must inform watch1 that it now has no resources. + // (SotW only: delta's explicit removals avoid the need for this guessing.) + expectEmptySotwNoDeltaUpdate(callbacks1, "version3"); doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version3"); } // Second watch loses interest. @@ -257,9 +265,9 @@ TEST(WatchMapTest, AddRemoveAdd) { EXPECT_TRUE(added_removed.removed_.empty()); watch_map.updateWatchInterest(watch2, {"dummy"}); - // First watch receives update. + // *Only* first watch receives update. expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version1"); - expectNoDeltaUpdate(callbacks2, "version1"); + expectNoUpdate(callbacks2, "version1"); doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); } // First watch loses interest. @@ -278,9 +286,13 @@ TEST(WatchMapTest, AddRemoveAdd) { EXPECT_EQ(std::set({"alice"}), added_removed.added_); // add to subscription EXPECT_TRUE(added_removed.removed_.empty()); - // *Only* second watch receives update. - expectNoDeltaUpdate(callbacks1, "version2"); + // Both watches receive the update. For watch2, this is obviously desired. expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version2"); + // For watch1, it's more subtle: the WatchMap sees that this update has no + // resources watch1 cares about, but also knows that watch1 previously had + // some resources. So, it must inform watch1 that it now has no resources. + // (SotW only: delta's explicit removals avoid the need for this guessing.) + expectEmptySotwNoDeltaUpdate(callbacks1, "version2"); doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version2"); } } @@ -302,14 +314,20 @@ TEST(WatchMapTest, UninterestingUpdate) { bob.set_cluster_name("bob"); bob_update.Add()->PackFrom(bob); - expectNoDeltaUpdate(callbacks, "version1"); + // We are watching for alice, and an update for just bob arrives. It should be ignored. + expectNoUpdate(callbacks, "version1"); doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version1"); + ::testing::Mock::VerifyAndClearExpectations(&callbacks); + // The server sends an update adding alice and removing bob. We pay attention only to alice. expectDeltaAndSotwUpdate(callbacks, {alice}, {}, "version2"); doDeltaAndSotwUpdate(watch_map, alice_update, {}, "version2"); + ::testing::Mock::VerifyAndClearExpectations(&callbacks); - expectNoDeltaUpdate(callbacks, "version3"); - doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version3"); + // The server sends an update removing alice and adding bob. We pay attention only to alice. + expectDeltaAndSotwUpdate(callbacks, {}, {"alice"}, "version3"); + doDeltaAndSotwUpdate(watch_map, bob_update, {"alice"}, "version3"); + ::testing::Mock::VerifyAndClearExpectations(&callbacks); // Clean removal of the watch: first update to "interested in nothing", then remove. watch_map.updateWatchInterest(watch, {}); @@ -366,6 +384,19 @@ TEST(WatchMapTest, DeltaOnConfigUpdate) { watch_map.updateWatchInterest(watch2, {"updated", "removed"}); watch_map.updateWatchInterest(watch3, {"removed"}); + // First, create the "removed" resource. We want to test SotW being handed an empty + // onConfigUpdate. But, if SotW holds no resources, then an update with nothing it cares about + // will just not trigger any onConfigUpdate at all. + { + Protobuf::RepeatedPtrField prepare_removed; + envoy::api::v2::ClusterLoadAssignment will_be_removed_later; + will_be_removed_later.set_cluster_name("removed"); + prepare_removed.Add()->PackFrom(will_be_removed_later); + expectDeltaAndSotwUpdate(callbacks2, {will_be_removed_later}, {}, "version0"); + expectDeltaAndSotwUpdate(callbacks3, {will_be_removed_later}, {}, "version0"); + doDeltaAndSotwUpdate(watch_map, prepare_removed, {}, "version0"); + } + Protobuf::RepeatedPtrField update; envoy::api::v2::ClusterLoadAssignment updated; updated.set_cluster_name("updated"); diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 0b49c5a9af85..9204d82d1893 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -333,6 +333,7 @@ accessors acls addr agg +alice alignas alignof alloc From eadd63f7be4cc0a45872fb1c39cd552eefefe894 Mon Sep 17 00:00:00 2001 From: asraa Date: Thu, 26 Sep 2019 18:31:16 -0400 Subject: [PATCH 09/32] fuzz: fix oss-fuzz build (#8401) Update rules_foreign_cc to include bazelbuild/rules_foreign_cc#321 This allows curl to be built with Envoy's libraries for dependencies (c-ares, zlib, ssl) without sandboxing. Testing: OSS-Fuzz builds pointed to this branch Signed-off-by: Asra Ali --- bazel/repository_locations.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 07674ee1bf08..9abaf46a9ecf 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -229,10 +229,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.19.2/rules_go-0.19.2.tar.gz"], ), rules_foreign_cc = dict( - sha256 = "ea7e4f13b66009d46c01a0292cf1d590d8ea06775c315263abb66022dde25315", - strip_prefix = "rules_foreign_cc-16ddc00bd4e1b3daf3faee1605a168f5283326fa", - # 2019-09-18 - urls = ["https://github.com/bazelbuild/rules_foreign_cc/archive/16ddc00bd4e1b3daf3faee1605a168f5283326fa.tar.gz"], + sha256 = "3184c244b32e65637a74213fc448964b687390eeeca42a36286f874c046bba15", + strip_prefix = "rules_foreign_cc-7bc4be735b0560289f6b86ab6136ee25d20b65b7", + # 2019-09-26 + urls = ["https://github.com/bazelbuild/rules_foreign_cc/archive/7bc4be735b0560289f6b86ab6136ee25d20b65b7.tar.gz"], ), rules_proto = dict( sha256 = "602e7161d9195e50246177e7c55b2f39950a9cf7366f74ed5f22fd45750cd208", From 3068ba961c36456f9e0c0e9d5a1d279982308933 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Thu, 26 Sep 2019 15:53:27 -0700 Subject: [PATCH 10/32] stats: create StatNameSet via SymbolTable::makeSet rather than construction. (#8369) Signed-off-by: Joshua Marantz --- include/envoy/stats/symbol_table.h | 19 ++++++++ source/common/stats/fake_symbol_table_impl.h | 6 +++ source/common/stats/symbol_table_impl.cc | 19 +++++++- source/common/stats/symbol_table_impl.h | 24 +++++++++- .../filters/http/dynamo/dynamo_stats.cc | 48 +++++++++---------- .../filters/http/dynamo/dynamo_stats.h | 6 +-- .../filters/http/fault/fault_filter.cc | 11 +++-- .../filters/http/fault/fault_filter.h | 2 +- .../http/ip_tagging/ip_tagging_filter.cc | 10 ++-- .../http/ip_tagging/ip_tagging_filter.h | 2 +- .../common/redis/redis_command_stats.cc | 28 +++++------ .../common/redis/redis_command_stats.h | 2 +- .../network/mongo_proxy/mongo_stats.cc | 20 ++++---- .../filters/network/mongo_proxy/mongo_stats.h | 6 +-- .../filters/network/zookeeper_proxy/filter.cc | 18 +++---- .../filters/network/zookeeper_proxy/filter.h | 2 +- .../transport_sockets/tls/context_impl.cc | 39 +++++++-------- .../transport_sockets/tls/context_impl.h | 2 +- test/common/stats/symbol_table_impl_test.cc | 22 ++++----- 19 files changed, 177 insertions(+), 109 deletions(-) diff --git a/include/envoy/stats/symbol_table.h b/include/envoy/stats/symbol_table.h index a4346d69397b..204cc227cb91 100644 --- a/include/envoy/stats/symbol_table.h +++ b/include/envoy/stats/symbol_table.h @@ -22,6 +22,9 @@ class StatName; using StatNameVec = std::vector; class StatNameList; +class StatNameSet; + +using StatNameSetPtr = std::unique_ptr; /** * SymbolTable manages a namespace optimized for stat names, exploiting their @@ -143,10 +146,19 @@ class SymbolTable { virtual void callWithStringView(StatName stat_name, const std::function& fn) const PURE; + /** + * Creates a StatNameSet. + * + * @param name the name of the set. + * @return the set. + */ + virtual StatNameSetPtr makeSet(absl::string_view name) PURE; + private: friend struct HeapStatData; friend class StatNameStorage; friend class StatNameList; + friend class StatNameSet; // The following methods are private, but are called by friend classes // StatNameStorage and StatNameList, which must be friendly with SymbolTable @@ -183,6 +195,13 @@ class SymbolTable { * */ virtual StoragePtr encode(absl::string_view name) PURE; + + /** + * Called by StatNameSet's destructor. + * + * @param stat_name_set the set. + */ + virtual void forgetSet(StatNameSet& stat_name_set) PURE; }; using SymbolTablePtr = std::unique_ptr; diff --git a/source/common/stats/fake_symbol_table_impl.h b/source/common/stats/fake_symbol_table_impl.h index 096cb42d4d72..8c43dc40b730 100644 --- a/source/common/stats/fake_symbol_table_impl.h +++ b/source/common/stats/fake_symbol_table_impl.h @@ -125,6 +125,12 @@ class FakeSymbolTableImpl : public SymbolTable { fn(toStringView(stat_name)); } + StatNameSetPtr makeSet(absl::string_view name) override { + // make_unique does not work with private ctor, even though FakeSymbolTableImpl is a friend. + return StatNameSetPtr(new StatNameSet(*this, name)); + } + void forgetSet(StatNameSet&) override {} + private: absl::string_view toStringView(const StatName& stat_name) const { return {reinterpret_cast(stat_name.data()), diff --git a/source/common/stats/symbol_table_impl.cc b/source/common/stats/symbol_table_impl.cc index a808bddfac83..b0ea68d88890 100644 --- a/source/common/stats/symbol_table_impl.cc +++ b/source/common/stats/symbol_table_impl.cc @@ -206,6 +206,20 @@ void SymbolTableImpl::free(const StatName& stat_name) { } } } + +StatNameSetPtr SymbolTableImpl::makeSet(absl::string_view name) { + Thread::LockGuard lock(lock_); + // make_unique does not work with private ctor, even though FakeSymbolTableImpl is a friend. + StatNameSetPtr stat_name_set(new StatNameSet(*this, name)); + stat_name_sets_.insert(stat_name_set.get()); + return stat_name_set; +} + +void SymbolTableImpl::forgetSet(StatNameSet& stat_name_set) { + Thread::LockGuard lock(lock_); + stat_name_sets_.erase(&stat_name_set); +} + Symbol SymbolTableImpl::toSymbol(absl::string_view sv) { Symbol result; auto encode_find = encode_map_.find(sv); @@ -429,10 +443,13 @@ void StatNameList::clear(SymbolTable& symbol_table) { storage_.reset(); } -StatNameSet::StatNameSet(SymbolTable& symbol_table) : pool_(symbol_table) { +StatNameSet::StatNameSet(SymbolTable& symbol_table, absl::string_view name) + : name_(std::string(name)), symbol_table_(symbol_table), pool_(symbol_table) { builtin_stat_names_[""] = StatName(); } +StatNameSet::~StatNameSet() { symbol_table_.forgetSet(*this); } + void StatNameSet::rememberBuiltin(absl::string_view str) { StatName stat_name; { diff --git a/source/common/stats/symbol_table_impl.h b/source/common/stats/symbol_table_impl.h index 36cc04166765..d047708fe8d9 100644 --- a/source/common/stats/symbol_table_impl.h +++ b/source/common/stats/symbol_table_impl.h @@ -159,6 +159,9 @@ class SymbolTableImpl : public SymbolTable { return bytes; } + StatNameSetPtr makeSet(absl::string_view name) override; + void forgetSet(StatNameSet& stat_name_set) override; + private: friend class StatName; friend class StatNameTest; @@ -238,6 +241,7 @@ class SymbolTableImpl : public SymbolTable { // TODO(ambuc): There might be an optimization here relating to storing ranges of freed symbols // using an Envoy::IntervalSet. std::stack pool_ GUARDED_BY(lock_); + absl::flat_hash_set stat_name_sets_ GUARDED_BY(lock_); }; /** @@ -647,7 +651,9 @@ class StatNameStorageSet { // builtins must *not* be added in the request-path. class StatNameSet { public: - explicit StatNameSet(SymbolTable& symbol_table); + // This object must be instantiated via SymbolTable::makeSet(), thus constructor is private. + + ~StatNameSet(); /** * Adds a string to the builtin map, which is not mutex protected. This map is @@ -679,11 +685,20 @@ class StatNameSet { * subsequent lookups of the same string to take only the set's lock, and not * the whole symbol-table lock. * + * @return a StatName corresponding to the passed-in token, owned by the set. + * * TODO(jmarantz): Potential perf issue here with contention, both on this * set's mutex and also the SymbolTable mutex which must be taken during * StatNamePool::add(). */ StatName getDynamic(absl::string_view token); + + /** + * Finds a builtin StatName by name. If the builtin has not been registered, + * then the fallback is returned. + * + * @return the StatName or fallback. + */ StatName getBuiltin(absl::string_view token, StatName fallback); /** @@ -695,6 +710,13 @@ class StatNameSet { } private: + friend class FakeSymbolTableImpl; + friend class SymbolTableImpl; + + StatNameSet(SymbolTable& symbol_table, absl::string_view name); + + const std::string name_; + Stats::SymbolTable& symbol_table_; Stats::StatNamePool pool_ GUARDED_BY(mutex_); absl::Mutex mutex_; using StringStatNameMap = absl::flat_hash_map; diff --git a/source/extensions/filters/http/dynamo/dynamo_stats.cc b/source/extensions/filters/http/dynamo/dynamo_stats.cc index 53e51133394c..29e2ef1d00bc 100644 --- a/source/extensions/filters/http/dynamo/dynamo_stats.cc +++ b/source/extensions/filters/http/dynamo/dynamo_stats.cc @@ -15,35 +15,35 @@ namespace HttpFilters { namespace Dynamo { DynamoStats::DynamoStats(Stats::Scope& scope, const std::string& prefix) - : scope_(scope), stat_name_set_(scope.symbolTable()), - prefix_(stat_name_set_.add(prefix + "dynamodb")), - batch_failure_unprocessed_keys_(stat_name_set_.add("BatchFailureUnprocessedKeys")), - capacity_(stat_name_set_.add("capacity")), - empty_response_body_(stat_name_set_.add("empty_response_body")), - error_(stat_name_set_.add("error")), - invalid_req_body_(stat_name_set_.add("invalid_req_body")), - invalid_resp_body_(stat_name_set_.add("invalid_resp_body")), - multiple_tables_(stat_name_set_.add("multiple_tables")), - no_table_(stat_name_set_.add("no_table")), - operation_missing_(stat_name_set_.add("operation_missing")), - table_(stat_name_set_.add("table")), table_missing_(stat_name_set_.add("table_missing")), - upstream_rq_time_(stat_name_set_.add("upstream_rq_time")), - upstream_rq_total_(stat_name_set_.add("upstream_rq_total")), - unknown_entity_type_(stat_name_set_.add("unknown_entity_type")), - unknown_operation_(stat_name_set_.add("unknown_operation")) { - upstream_rq_total_groups_[0] = stat_name_set_.add("upstream_rq_total_unknown"); - upstream_rq_time_groups_[0] = stat_name_set_.add("upstream_rq_time_unknown"); + : scope_(scope), stat_name_set_(scope.symbolTable().makeSet("Dynamo")), + prefix_(stat_name_set_->add(prefix + "dynamodb")), + batch_failure_unprocessed_keys_(stat_name_set_->add("BatchFailureUnprocessedKeys")), + capacity_(stat_name_set_->add("capacity")), + empty_response_body_(stat_name_set_->add("empty_response_body")), + error_(stat_name_set_->add("error")), + invalid_req_body_(stat_name_set_->add("invalid_req_body")), + invalid_resp_body_(stat_name_set_->add("invalid_resp_body")), + multiple_tables_(stat_name_set_->add("multiple_tables")), + no_table_(stat_name_set_->add("no_table")), + operation_missing_(stat_name_set_->add("operation_missing")), + table_(stat_name_set_->add("table")), table_missing_(stat_name_set_->add("table_missing")), + upstream_rq_time_(stat_name_set_->add("upstream_rq_time")), + upstream_rq_total_(stat_name_set_->add("upstream_rq_total")), + unknown_entity_type_(stat_name_set_->add("unknown_entity_type")), + unknown_operation_(stat_name_set_->add("unknown_operation")) { + upstream_rq_total_groups_[0] = stat_name_set_->add("upstream_rq_total_unknown"); + upstream_rq_time_groups_[0] = stat_name_set_->add("upstream_rq_time_unknown"); for (size_t i = 1; i < DynamoStats::NumGroupEntries; ++i) { - upstream_rq_total_groups_[i] = stat_name_set_.add(fmt::format("upstream_rq_total_{}xx", i)); - upstream_rq_time_groups_[i] = stat_name_set_.add(fmt::format("upstream_rq_time_{}xx", i)); + upstream_rq_total_groups_[i] = stat_name_set_->add(fmt::format("upstream_rq_total_{}xx", i)); + upstream_rq_time_groups_[i] = stat_name_set_->add(fmt::format("upstream_rq_time_{}xx", i)); } RequestParser::forEachStatString( - [this](const std::string& str) { stat_name_set_.rememberBuiltin(str); }); + [this](const std::string& str) { stat_name_set_->rememberBuiltin(str); }); for (uint32_t status_code : {200, 400, 403, 502}) { - stat_name_set_.rememberBuiltin(absl::StrCat("upstream_rq_time_", status_code)); - stat_name_set_.rememberBuiltin(absl::StrCat("upstream_rq_total_", status_code)); + stat_name_set_->rememberBuiltin(absl::StrCat("upstream_rq_time_", status_code)); + stat_name_set_->rememberBuiltin(absl::StrCat("upstream_rq_total_", status_code)); } - stat_name_set_.rememberBuiltins({"operation", "table"}); + stat_name_set_->rememberBuiltins({"operation", "table"}); } Stats::SymbolTable::StoragePtr DynamoStats::addPrefix(const Stats::StatNameVec& names) { diff --git a/source/extensions/filters/http/dynamo/dynamo_stats.h b/source/extensions/filters/http/dynamo/dynamo_stats.h index de5fab8ff523..8b802592877e 100644 --- a/source/extensions/filters/http/dynamo/dynamo_stats.h +++ b/source/extensions/filters/http/dynamo/dynamo_stats.h @@ -37,16 +37,16 @@ class DynamoStats { * TODO(jmarantz): Potential perf issue here with mutex contention for names * that have not been remembered as builtins in the constructor. */ - Stats::StatName getDynamic(const std::string& str) { return stat_name_set_.getDynamic(str); } + Stats::StatName getDynamic(const std::string& str) { return stat_name_set_->getDynamic(str); } Stats::StatName getBuiltin(const std::string& str, Stats::StatName fallback) { - return stat_name_set_.getBuiltin(str, fallback); + return stat_name_set_->getBuiltin(str, fallback); } private: Stats::SymbolTable::StoragePtr addPrefix(const Stats::StatNameVec& names); Stats::Scope& scope_; - Stats::StatNameSet stat_name_set_; + Stats::StatNameSetPtr stat_name_set_; const Stats::StatName prefix_; public: diff --git a/source/extensions/filters/http/fault/fault_filter.cc b/source/extensions/filters/http/fault/fault_filter.cc index 65d1211f6623..3fcc408f4de2 100644 --- a/source/extensions/filters/http/fault/fault_filter.cc +++ b/source/extensions/filters/http/fault/fault_filter.cc @@ -78,15 +78,16 @@ FaultFilterConfig::FaultFilterConfig(const envoy::config::filter::http::fault::v Runtime::Loader& runtime, const std::string& stats_prefix, Stats::Scope& scope, TimeSource& time_source) : settings_(fault), runtime_(runtime), stats_(generateStats(stats_prefix, scope)), - scope_(scope), time_source_(time_source), stat_name_set_(scope.symbolTable()), - aborts_injected_(stat_name_set_.add("aborts_injected")), - delays_injected_(stat_name_set_.add("delays_injected")), - stats_prefix_(stat_name_set_.add(absl::StrCat(stats_prefix, "fault"))) {} + scope_(scope), time_source_(time_source), + stat_name_set_(scope.symbolTable().makeSet("Fault")), + aborts_injected_(stat_name_set_->add("aborts_injected")), + delays_injected_(stat_name_set_->add("delays_injected")), + stats_prefix_(stat_name_set_->add(absl::StrCat(stats_prefix, "fault"))) {} void FaultFilterConfig::incCounter(absl::string_view downstream_cluster, Stats::StatName stat_name) { Stats::SymbolTable::StoragePtr storage = scope_.symbolTable().join( - {stats_prefix_, stat_name_set_.getDynamic(downstream_cluster), stat_name}); + {stats_prefix_, stat_name_set_->getDynamic(downstream_cluster), stat_name}); scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); } diff --git a/source/extensions/filters/http/fault/fault_filter.h b/source/extensions/filters/http/fault/fault_filter.h index b0d3a0f1bf30..066e2e3fb869 100644 --- a/source/extensions/filters/http/fault/fault_filter.h +++ b/source/extensions/filters/http/fault/fault_filter.h @@ -133,7 +133,7 @@ class FaultFilterConfig { FaultFilterStats stats_; Stats::Scope& scope_; TimeSource& time_source_; - Stats::StatNameSet stat_name_set_; + Stats::StatNameSetPtr stat_name_set_; const Stats::StatName aborts_injected_; const Stats::StatName delays_injected_; const Stats::StatName stats_prefix_; // Includes ".fault". diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc index 3d739e8f7933..fe3648c082de 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.cc @@ -14,10 +14,10 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( const envoy::config::filter::http::ip_tagging::v2::IPTagging& config, const std::string& stat_prefix, Stats::Scope& scope, Runtime::Loader& runtime) : request_type_(requestTypeEnum(config.request_type())), scope_(scope), runtime_(runtime), - stat_name_set_(scope.symbolTable()), - stats_prefix_(stat_name_set_.add(stat_prefix + "ip_tagging")), - hit_(stat_name_set_.add("hit")), no_hit_(stat_name_set_.add("no_hit")), - total_(stat_name_set_.add("total")) { + stat_name_set_(scope.symbolTable().makeSet("IpTagging")), + stats_prefix_(stat_name_set_->add(stat_prefix + "ip_tagging")), + hit_(stat_name_set_->add("hit")), no_hit_(stat_name_set_->add("no_hit")), + total_(stat_name_set_->add("total")) { // Once loading IP tags from a file system is supported, the restriction on the size // of the set should be removed and observability into what tags are loaded needs @@ -52,7 +52,7 @@ IpTaggingFilterConfig::IpTaggingFilterConfig( void IpTaggingFilterConfig::incCounter(Stats::StatName name, absl::string_view tag) { Stats::SymbolTable::StoragePtr storage = - scope_.symbolTable().join({stats_prefix_, stat_name_set_.getDynamic(tag), name}); + scope_.symbolTable().join({stats_prefix_, stat_name_set_->getDynamic(tag), name}); scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); } diff --git a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h index 6cf2b19b0e74..95ab46909682 100644 --- a/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h +++ b/source/extensions/filters/http/ip_tagging/ip_tagging_filter.h @@ -64,7 +64,7 @@ class IpTaggingFilterConfig { const FilterRequestType request_type_; Stats::Scope& scope_; Runtime::Loader& runtime_; - Stats::StatNameSet stat_name_set_; + Stats::StatNameSetPtr stat_name_set_; const Stats::StatName stats_prefix_; const Stats::StatName hit_; const Stats::StatName no_hit_; diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.cc b/source/extensions/filters/network/common/redis/redis_command_stats.cc index 6eb6967c7644..6178f363c6d2 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.cc +++ b/source/extensions/filters/network/common/redis/redis_command_stats.cc @@ -9,24 +9,24 @@ namespace Common { namespace Redis { RedisCommandStats::RedisCommandStats(Stats::SymbolTable& symbol_table, const std::string& prefix) - : symbol_table_(symbol_table), stat_name_set_(symbol_table_), - prefix_(stat_name_set_.add(prefix)), - upstream_rq_time_(stat_name_set_.add("upstream_rq_time")), - latency_(stat_name_set_.add("latency")), total_(stat_name_set_.add("total")), - success_(stat_name_set_.add("success")), error_(stat_name_set_.add("error")), - unused_metric_(stat_name_set_.add("unused")), null_metric_(stat_name_set_.add("null")), - unknown_metric_(stat_name_set_.add("unknown")) { + : symbol_table_(symbol_table), stat_name_set_(symbol_table_.makeSet("Redis")), + prefix_(stat_name_set_->add(prefix)), + upstream_rq_time_(stat_name_set_->add("upstream_rq_time")), + latency_(stat_name_set_->add("latency")), total_(stat_name_set_->add("total")), + success_(stat_name_set_->add("success")), error_(stat_name_set_->add("error")), + unused_metric_(stat_name_set_->add("unused")), null_metric_(stat_name_set_->add("null")), + unknown_metric_(stat_name_set_->add("unknown")) { // Note: Even if this is disabled, we track the upstream_rq_time. // Create StatName for each Redis command. Note that we don't include Auth or Ping. - stat_name_set_.rememberBuiltins( + stat_name_set_->rememberBuiltins( Extensions::NetworkFilters::Common::Redis::SupportedCommands::simpleCommands()); - stat_name_set_.rememberBuiltins( + stat_name_set_->rememberBuiltins( Extensions::NetworkFilters::Common::Redis::SupportedCommands::evalCommands()); - stat_name_set_.rememberBuiltins(Extensions::NetworkFilters::Common::Redis::SupportedCommands:: - hashMultipleSumResultCommands()); - stat_name_set_.rememberBuiltin( + stat_name_set_->rememberBuiltins(Extensions::NetworkFilters::Common::Redis::SupportedCommands:: + hashMultipleSumResultCommands()); + stat_name_set_->rememberBuiltin( Extensions::NetworkFilters::Common::Redis::SupportedCommands::mget()); - stat_name_set_.rememberBuiltin( + stat_name_set_->rememberBuiltin( Extensions::NetworkFilters::Common::Redis::SupportedCommands::mset()); } @@ -69,7 +69,7 @@ Stats::StatName RedisCommandStats::getCommandFromRequest(const RespValue& reques default: std::string to_lower_command(request.asString()); to_lower_table_.toLowerCase(to_lower_command); - return stat_name_set_.getBuiltin(to_lower_command, unknown_metric_); + return stat_name_set_->getBuiltin(to_lower_command, unknown_metric_); } } diff --git a/source/extensions/filters/network/common/redis/redis_command_stats.h b/source/extensions/filters/network/common/redis/redis_command_stats.h index b4b8c87a29db..556cc00e0150 100644 --- a/source/extensions/filters/network/common/redis/redis_command_stats.h +++ b/source/extensions/filters/network/common/redis/redis_command_stats.h @@ -42,7 +42,7 @@ class RedisCommandStats { private: Stats::SymbolTable& symbol_table_; - Stats::StatNameSet stat_name_set_; + Stats::StatNameSetPtr stat_name_set_; const Stats::StatName prefix_; const Stats::StatName upstream_rq_time_; const Stats::StatName latency_; diff --git a/source/extensions/filters/network/mongo_proxy/mongo_stats.cc b/source/extensions/filters/network/mongo_proxy/mongo_stats.cc index f82ca1a41f11..d23de1b6f49b 100644 --- a/source/extensions/filters/network/mongo_proxy/mongo_stats.cc +++ b/source/extensions/filters/network/mongo_proxy/mongo_stats.cc @@ -14,19 +14,21 @@ namespace NetworkFilters { namespace MongoProxy { MongoStats::MongoStats(Stats::Scope& scope, const std::string& prefix) - : scope_(scope), stat_name_set_(scope.symbolTable()), prefix_(stat_name_set_.add(prefix)), - callsite_(stat_name_set_.add("callsite")), cmd_(stat_name_set_.add("cmd")), - collection_(stat_name_set_.add("collection")), multi_get_(stat_name_set_.add("multi_get")), - reply_num_docs_(stat_name_set_.add("reply_num_docs")), - reply_size_(stat_name_set_.add("reply_size")), - reply_time_ms_(stat_name_set_.add("reply_time_ms")), time_ms_(stat_name_set_.add("time_ms")), - query_(stat_name_set_.add("query")), scatter_get_(stat_name_set_.add("scatter_get")), - total_(stat_name_set_.add("total")), unknown_command_(stat_name_set_.add("unknown_command")) { + : scope_(scope), stat_name_set_(scope.symbolTable().makeSet("Mongo")), + prefix_(stat_name_set_->add(prefix)), callsite_(stat_name_set_->add("callsite")), + cmd_(stat_name_set_->add("cmd")), collection_(stat_name_set_->add("collection")), + multi_get_(stat_name_set_->add("multi_get")), + reply_num_docs_(stat_name_set_->add("reply_num_docs")), + reply_size_(stat_name_set_->add("reply_size")), + reply_time_ms_(stat_name_set_->add("reply_time_ms")), + time_ms_(stat_name_set_->add("time_ms")), query_(stat_name_set_->add("query")), + scatter_get_(stat_name_set_->add("scatter_get")), total_(stat_name_set_->add("total")), + unknown_command_(stat_name_set_->add("unknown_command")) { // TODO(jmarantz): is this the right set of mongo commands to use as builtins? // Should we also have builtins for callsites or collections, or do those need // to be dynamic? - stat_name_set_.rememberBuiltins({"insert", "query", "update", "delete"}); + stat_name_set_->rememberBuiltins({"insert", "query", "update", "delete"}); } Stats::SymbolTable::StoragePtr MongoStats::addPrefix(const std::vector& names) { diff --git a/source/extensions/filters/network/mongo_proxy/mongo_stats.h b/source/extensions/filters/network/mongo_proxy/mongo_stats.h index e4cb0426ba00..aeb657f13526 100644 --- a/source/extensions/filters/network/mongo_proxy/mongo_stats.h +++ b/source/extensions/filters/network/mongo_proxy/mongo_stats.h @@ -27,16 +27,16 @@ class MongoStats { * that have not been remembered as builtins in the constructor. */ Stats::StatName getBuiltin(const std::string& str, Stats::StatName fallback) { - return stat_name_set_.getBuiltin(str, fallback); + return stat_name_set_->getBuiltin(str, fallback); } - Stats::StatName getDynamic(const std::string& str) { return stat_name_set_.getDynamic(str); } + Stats::StatName getDynamic(const std::string& str) { return stat_name_set_->getDynamic(str); } private: Stats::SymbolTable::StoragePtr addPrefix(const std::vector& names); Stats::Scope& scope_; - Stats::StatNameSet stat_name_set_; + Stats::StatNameSetPtr stat_name_set_; public: const Stats::StatName prefix_; diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.cc b/source/extensions/filters/network/zookeeper_proxy/filter.cc index 2fee4392cc5b..bae059cf2757 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.cc +++ b/source/extensions/filters/network/zookeeper_proxy/filter.cc @@ -19,15 +19,15 @@ namespace ZooKeeperProxy { ZooKeeperFilterConfig::ZooKeeperFilterConfig(const std::string& stat_prefix, const uint32_t max_packet_bytes, Stats::Scope& scope) : scope_(scope), max_packet_bytes_(max_packet_bytes), stats_(generateStats(stat_prefix, scope)), - stat_name_set_(scope.symbolTable()), stat_prefix_(stat_name_set_.add(stat_prefix)), - auth_(stat_name_set_.add("auth")), - connect_latency_(stat_name_set_.add("connect_response_latency")), - unknown_scheme_rq_(stat_name_set_.add("unknown_scheme_rq")), - unknown_opcode_latency_(stat_name_set_.add("unknown_opcode_latency")) { + stat_name_set_(scope.symbolTable().makeSet("Zookeeper")), + stat_prefix_(stat_name_set_->add(stat_prefix)), auth_(stat_name_set_->add("auth")), + connect_latency_(stat_name_set_->add("connect_response_latency")), + unknown_scheme_rq_(stat_name_set_->add("unknown_scheme_rq")), + unknown_opcode_latency_(stat_name_set_->add("unknown_opcode_latency")) { // https://zookeeper.apache.org/doc/r3.5.4-beta/zookeeperProgrammers.html#sc_BuiltinACLSchemes // lists commons schemes: "world", "auth", "digest", "host", "x509", and // "ip". These are used in filter.cc by appending "_rq". - stat_name_set_.rememberBuiltins( + stat_name_set_->rememberBuiltins( {"auth_rq", "digest_rq", "host_rq", "ip_rq", "ping_response_rq", "world_rq", "x509_rq"}); initOpCode(OpCodes::PING, stats_.ping_resp_, "ping_response"); @@ -62,7 +62,7 @@ void ZooKeeperFilterConfig::initOpCode(OpCodes opcode, Stats::Counter& counter, OpCodeInfo& opcode_info = op_code_map_[opcode]; opcode_info.counter_ = &counter; opcode_info.opname_ = std::string(name); - opcode_info.latency_name_ = stat_name_set_.add(absl::StrCat(name, "_latency")); + opcode_info.latency_name_ = stat_name_set_->add(absl::StrCat(name, "_latency")); } ZooKeeperFilter::ZooKeeperFilter(ZooKeeperFilterConfigSharedPtr config, TimeSource& time_source) @@ -153,8 +153,8 @@ void ZooKeeperFilter::onPing() { void ZooKeeperFilter::onAuthRequest(const std::string& scheme) { Stats::SymbolTable::StoragePtr storage = config_->scope_.symbolTable().join( {config_->stat_prefix_, config_->auth_, - config_->stat_name_set_.getBuiltin(absl::StrCat(scheme, "_rq"), - config_->unknown_scheme_rq_)}); + config_->stat_name_set_->getBuiltin(absl::StrCat(scheme, "_rq"), + config_->unknown_scheme_rq_)}); config_->scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); setDynamicMetadata("opname", "auth"); } diff --git a/source/extensions/filters/network/zookeeper_proxy/filter.h b/source/extensions/filters/network/zookeeper_proxy/filter.h index 14d6ca1b9a89..56b9d858c35b 100644 --- a/source/extensions/filters/network/zookeeper_proxy/filter.h +++ b/source/extensions/filters/network/zookeeper_proxy/filter.h @@ -116,7 +116,7 @@ class ZooKeeperFilterConfig { Stats::Scope& scope_; const uint32_t max_packet_bytes_; ZooKeeperProxyStats stats_; - Stats::StatNameSet stat_name_set_; + Stats::StatNameSetPtr stat_name_set_; const Stats::StatName stat_prefix_; const Stats::StatName auth_; const Stats::StatName connect_latency_; diff --git a/source/extensions/transport_sockets/tls/context_impl.cc b/source/extensions/transport_sockets/tls/context_impl.cc index fc4543e56df5..ede655319cb2 100644 --- a/source/extensions/transport_sockets/tls/context_impl.cc +++ b/source/extensions/transport_sockets/tls/context_impl.cc @@ -51,15 +51,16 @@ bool cbsContainsU16(CBS& cbs, uint16_t n) { ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& config, TimeSource& time_source) : scope_(scope), stats_(generateStats(scope)), time_source_(time_source), - tls_max_version_(config.maxProtocolVersion()), stat_name_set_(scope.symbolTable()), - unknown_ssl_cipher_(stat_name_set_.add("unknown_ssl_cipher")), - unknown_ssl_curve_(stat_name_set_.add("unknown_ssl_curve")), - unknown_ssl_algorithm_(stat_name_set_.add("unknown_ssl_algorithm")), - unknown_ssl_version_(stat_name_set_.add("unknown_ssl_version")), - ssl_ciphers_(stat_name_set_.add("ssl.ciphers")), - ssl_versions_(stat_name_set_.add("ssl.versions")), - ssl_curves_(stat_name_set_.add("ssl.curves")), - ssl_sigalgs_(stat_name_set_.add("ssl.sigalgs")) { + tls_max_version_(config.maxProtocolVersion()), + stat_name_set_(scope.symbolTable().makeSet("TransportSockets::Tls")), + unknown_ssl_cipher_(stat_name_set_->add("unknown_ssl_cipher")), + unknown_ssl_curve_(stat_name_set_->add("unknown_ssl_curve")), + unknown_ssl_algorithm_(stat_name_set_->add("unknown_ssl_algorithm")), + unknown_ssl_version_(stat_name_set_->add("unknown_ssl_version")), + ssl_ciphers_(stat_name_set_->add("ssl.ciphers")), + ssl_versions_(stat_name_set_->add("ssl.versions")), + ssl_curves_(stat_name_set_->add("ssl.curves")), + ssl_sigalgs_(stat_name_set_->add("ssl.sigalgs")) { const auto tls_certificates = config.tlsCertificates(); tls_contexts_.resize(std::max(static_cast(1), tls_certificates.size())); @@ -388,23 +389,23 @@ ContextImpl::ContextImpl(Stats::Scope& scope, const Envoy::Ssl::ContextConfig& c // contention. // Ciphers - stat_name_set_.rememberBuiltin("AEAD-AES128-GCM-SHA256"); - stat_name_set_.rememberBuiltin("ECDHE-ECDSA-AES128-GCM-SHA256"); - stat_name_set_.rememberBuiltin("ECDHE-RSA-AES128-GCM-SHA256"); - stat_name_set_.rememberBuiltin("ECDHE-RSA-AES128-SHA"); - stat_name_set_.rememberBuiltin("ECDHE-RSA-CHACHA20-POLY1305"); - stat_name_set_.rememberBuiltin("TLS_AES_128_GCM_SHA256"); + stat_name_set_->rememberBuiltin("AEAD-AES128-GCM-SHA256"); + stat_name_set_->rememberBuiltin("ECDHE-ECDSA-AES128-GCM-SHA256"); + stat_name_set_->rememberBuiltin("ECDHE-RSA-AES128-GCM-SHA256"); + stat_name_set_->rememberBuiltin("ECDHE-RSA-AES128-SHA"); + stat_name_set_->rememberBuiltin("ECDHE-RSA-CHACHA20-POLY1305"); + stat_name_set_->rememberBuiltin("TLS_AES_128_GCM_SHA256"); // Curves from // https://github.com/google/boringssl/blob/f4d8b969200f1ee2dd872ffb85802e6a0976afe7/ssl/ssl_key_share.cc#L384 - stat_name_set_.rememberBuiltins( + stat_name_set_->rememberBuiltins( {"P-224", "P-256", "P-384", "P-521", "X25519", "CECPQ2", "CECPQ2b"}); // Algorithms - stat_name_set_.rememberBuiltins({"ecdsa_secp256r1_sha256", "rsa_pss_rsae_sha256"}); + stat_name_set_->rememberBuiltins({"ecdsa_secp256r1_sha256", "rsa_pss_rsae_sha256"}); // Versions - stat_name_set_.rememberBuiltins({"TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"}); + stat_name_set_->rememberBuiltins({"TLSv1", "TLSv1.1", "TLSv1.2", "TLSv1.3"}); } int ServerContextImpl::alpnSelectCallback(const unsigned char** out, unsigned char* outlen, @@ -517,7 +518,7 @@ void ContextImpl::incCounter(const Stats::StatName name, absl::string_view value const Stats::StatName fallback) const { Stats::SymbolTable& symbol_table = scope_.symbolTable(); Stats::SymbolTable::StoragePtr storage = - symbol_table.join({name, stat_name_set_.getBuiltin(value, fallback)}); + symbol_table.join({name, stat_name_set_->getBuiltin(value, fallback)}); scope_.counterFromStatName(Stats::StatName(storage.get())).inc(); #ifdef LOG_BUILTIN_STAT_NAMES diff --git a/source/extensions/transport_sockets/tls/context_impl.h b/source/extensions/transport_sockets/tls/context_impl.h index 602ee16eddf4..f91ad1d0bc8f 100644 --- a/source/extensions/transport_sockets/tls/context_impl.h +++ b/source/extensions/transport_sockets/tls/context_impl.h @@ -171,7 +171,7 @@ class ContextImpl : public virtual Envoy::Ssl::Context { std::string cert_chain_file_path_; TimeSource& time_source_; const unsigned tls_max_version_; - mutable Stats::StatNameSet stat_name_set_; + mutable Stats::StatNameSetPtr stat_name_set_; const Stats::StatName unknown_ssl_cipher_; const Stats::StatName unknown_ssl_curve_; const Stats::StatName unknown_ssl_algorithm_; diff --git a/test/common/stats/symbol_table_impl_test.cc b/test/common/stats/symbol_table_impl_test.cc index bfd829bde3cd..6c7e5f32dbfc 100644 --- a/test/common/stats/symbol_table_impl_test.cc +++ b/test/common/stats/symbol_table_impl_test.cc @@ -539,32 +539,32 @@ TEST_P(StatNameTest, SharedStatNameStorageSetSwap) { } TEST_P(StatNameTest, StatNameSet) { - StatNameSet set(*table_); + StatNameSetPtr set(table_->makeSet("set")); // Test that we get a consistent StatName object from a remembered name. - set.rememberBuiltin("remembered"); - const StatName fallback = set.add("fallback"); - const Stats::StatName remembered = set.getBuiltin("remembered", fallback); + set->rememberBuiltin("remembered"); + const StatName fallback = set->add("fallback"); + const Stats::StatName remembered = set->getBuiltin("remembered", fallback); EXPECT_EQ("remembered", table_->toString(remembered)); - EXPECT_EQ(remembered.data(), set.getBuiltin("remembered", fallback).data()); - EXPECT_EQ(fallback.data(), set.getBuiltin("not_remembered", fallback).data()); + EXPECT_EQ(remembered.data(), set->getBuiltin("remembered", fallback).data()); + EXPECT_EQ(fallback.data(), set->getBuiltin("not_remembered", fallback).data()); // Same test for a dynamically allocated name. The only difference between // the behavior with a remembered vs dynamic name is that when looking // up a remembered name, a mutex is not taken. But we have no easy way // to test for that. So we'll at least cover the code. - const Stats::StatName dynamic = set.getDynamic("dynamic"); + const Stats::StatName dynamic = set->getDynamic("dynamic"); EXPECT_EQ("dynamic", table_->toString(dynamic)); - EXPECT_EQ(dynamic.data(), set.getDynamic("dynamic").data()); + EXPECT_EQ(dynamic.data(), set->getDynamic("dynamic").data()); // There's another corner case for the same "dynamic" name from a // different set. Here we will get a different StatName object // out of the second set, though it will share the same underlying // symbol-table symbol. - StatNameSet set2(*table_); - const Stats::StatName dynamic2 = set2.getDynamic("dynamic"); + StatNameSetPtr set2(table_->makeSet("set2")); + const Stats::StatName dynamic2 = set2->getDynamic("dynamic"); EXPECT_EQ("dynamic", table_->toString(dynamic2)); - EXPECT_EQ(dynamic2.data(), set2.getDynamic("dynamic").data()); + EXPECT_EQ(dynamic2.data(), set2->getDynamic("dynamic").data()); EXPECT_NE(dynamic2.data(), dynamic.data()); } From b0ab9fb8bb49819eed03265d18b744ac4b13e593 Mon Sep 17 00:00:00 2001 From: alyssawilk Date: Thu, 26 Sep 2019 18:56:25 -0400 Subject: [PATCH 11/32] http: fixing a resumption bug in pipelining (#8352) Signed-off-by: Alyssa Wilk --- source/common/http/conn_manager_impl.cc | 1 + source/common/network/connection_impl.cc | 17 +++-- source/common/network/connection_impl.h | 25 ++++--- test/common/network/connection_impl_test.cc | 81 +++++++++++++++++++++ test/integration/integration_test.cc | 26 +++++++ 5 files changed, 133 insertions(+), 17 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 8fb6149151e6..cdd68ebcc572 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -284,6 +284,7 @@ Network::FilterStatus ConnectionManagerImpl::onData(Buffer::Instance& data, bool } } + // TODO(alyssawilk) clean this up after #8352 is well vetted. bool redispatch; do { redispatch = false; diff --git a/source/common/network/connection_impl.cc b/source/common/network/connection_impl.cc index a5f1b4e7b8fe..17fba821a733 100644 --- a/source/common/network/connection_impl.cc +++ b/source/common/network/connection_impl.cc @@ -51,7 +51,10 @@ ConnectionImpl::ConnectionImpl(Event::Dispatcher& dispatcher, ConnectionSocketPt write_buffer_( dispatcher.getWatermarkFactory().create([this]() -> void { this->onLowWatermark(); }, [this]() -> void { this->onHighWatermark(); })), - dispatcher_(dispatcher), id_(next_global_id_++) { + dispatcher_(dispatcher), id_(next_global_id_++), read_enabled_(true), + above_high_watermark_(false), detect_early_close_(true), enable_half_close_(false), + read_end_stream_raised_(false), read_end_stream_(false), write_end_stream_(false), + current_write_end_stream_(false) { // Treat the lack of a valid fd (which in practice only happens if we run out of FDs) as an OOM // condition and just crash. RELEASE_ASSERT(ioHandle().fd() != -1, ""); @@ -323,8 +326,9 @@ void ConnectionImpl::readDisable(bool disable) { file_event_->setEnabled(Event::FileReadyType::Read | Event::FileReadyType::Write); // If the connection has data buffered there's no guarantee there's also data in the kernel // which will kick off the filter chain. Instead fake an event to make sure the buffered data - // gets processed regardless. + // gets processed regardless and ensure that we dispatch it via onRead. if (read_buffer_.length() > 0) { + dispatch_buffered_data_ = true; file_event_->activate(Event::FileReadyType::Read); } } @@ -509,11 +513,14 @@ void ConnectionImpl::onReadReady() { } read_end_stream_ |= result.end_stream_read_; - if (result.bytes_processed_ != 0 || result.end_stream_read_) { - // Skip onRead if no bytes were processed. For instance, if the connection was closed without - // producing more data. + if (result.bytes_processed_ != 0 || result.end_stream_read_ || + (dispatch_buffered_data_ && read_buffer_.length() > 0)) { + // Skip onRead if no bytes were processed unless we explicitly want to force onRead for + // buffered data. For instance, skip onRead if the connection was closed without producing + // more data. onRead(new_buffer_size); } + dispatch_buffered_data_ = false; // The read callback may have already closed the connection. if (result.action_ == PostIoAction::Close || bothSidesHalfClosed()) { diff --git a/source/common/network/connection_impl.h b/source/common/network/connection_impl.h index 5923951a34e5..547ed540a8cb 100644 --- a/source/common/network/connection_impl.h +++ b/source/common/network/connection_impl.h @@ -203,22 +203,23 @@ class ConnectionImpl : public FilterManagerConnection, Event::TimerPtr delayed_close_timer_; std::list callbacks_; std::list bytes_sent_callbacks_; - bool read_enabled_{true}; - bool above_high_watermark_{false}; - bool detect_early_close_{true}; - bool enable_half_close_{false}; - bool read_end_stream_raised_{false}; - bool read_end_stream_{false}; - bool write_end_stream_{false}; - bool current_write_end_stream_{false}; - Buffer::Instance* current_write_buffer_{}; - uint64_t last_read_buffer_size_{}; - uint64_t last_write_buffer_size_{}; - std::unique_ptr connection_stats_; // Tracks the number of times reads have been disabled. If N different components call // readDisabled(true) this allows the connection to only resume reads when readDisabled(false) // has been called N times. + uint64_t last_read_buffer_size_{}; + uint64_t last_write_buffer_size_{}; + std::unique_ptr connection_stats_; + Buffer::Instance* current_write_buffer_{}; uint32_t read_disable_count_{0}; + bool read_enabled_ : 1; + bool above_high_watermark_ : 1; + bool detect_early_close_ : 1; + bool enable_half_close_ : 1; + bool read_end_stream_raised_ : 1; + bool read_end_stream_ : 1; + bool write_end_stream_ : 1; + bool current_write_end_stream_ : 1; + bool dispatch_buffered_data_ : 1; }; /** diff --git a/test/common/network/connection_impl_test.cc b/test/common/network/connection_impl_test.cc index 7e22ba77e941..413909967856 100644 --- a/test/common/network/connection_impl_test.cc +++ b/test/common/network/connection_impl_test.cc @@ -509,6 +509,87 @@ TEST_P(ConnectionImplTest, ReadDisable) { disconnect(false); } +// The HTTP/1 codec handles pipelined connections by relying on readDisable(false) resulting in the +// subsequent request being dispatched. Regression test this behavior. +TEST_P(ConnectionImplTest, ReadEnableDispatches) { + setUpBasicConnection(); + connect(); + + std::shared_ptr client_read_filter(new NiceMock()); + client_connection_->addReadFilter(client_read_filter); + + { + Buffer::OwnedImpl buffer("data"); + server_connection_->write(buffer, false); + EXPECT_CALL(*client_read_filter, onData(BufferStringEqual("data"), false)) + .WillOnce(Invoke([&](Buffer::Instance&, bool) -> FilterStatus { + dispatcher_->exit(); + return FilterStatus::StopIteration; + })); + dispatcher_->run(Event::Dispatcher::RunType::Block); + } + + { + client_connection_->readDisable(true); + EXPECT_CALL(*client_read_filter, onData(BufferStringEqual("data"), false)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) -> FilterStatus { + buffer.drain(buffer.length()); + dispatcher_->exit(); + return FilterStatus::StopIteration; + })); + client_connection_->readDisable(false); + dispatcher_->run(Event::Dispatcher::RunType::Block); + } + + disconnect(true); +} + +// Make sure if we readDisable(true) and schedule a 'kick' and then +// readDisable(false) the kick doesn't happen. +TEST_P(ConnectionImplTest, KickUndone) { + setUpBasicConnection(); + connect(); + + std::shared_ptr client_read_filter(new NiceMock()); + client_connection_->addReadFilter(client_read_filter); + Buffer::Instance* connection_buffer = nullptr; + + { + Buffer::OwnedImpl buffer("data"); + server_connection_->write(buffer, false); + EXPECT_CALL(*client_read_filter, onData(BufferStringEqual("data"), false)) + .WillOnce(Invoke([&](Buffer::Instance& buffer, bool) -> FilterStatus { + dispatcher_->exit(); + connection_buffer = &buffer; + return FilterStatus::StopIteration; + })); + dispatcher_->run(Event::Dispatcher::RunType::Block); + } + + { + // Like ReadEnableDispatches above, read disable and read enable to kick off + // an extra read. But then readDisable again and make sure the kick doesn't + // happen. + client_connection_->readDisable(true); + client_connection_->readDisable(false); // Sets dispatch_buffered_data_ + client_connection_->readDisable(true); + EXPECT_CALL(*client_read_filter, onData(_, _)).Times(0); + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + // Now drain the connection's buffer and try to do a read which should _not_ + // pass up the stack (no data is read) + { + connection_buffer->drain(connection_buffer->length()); + client_connection_->readDisable(false); + EXPECT_CALL(*client_read_filter, onData(_, _)).Times(0); + // Data no longer buffered - even if dispatch_buffered_data_ lingered it should have no effect. + dispatcher_->run(Event::Dispatcher::RunType::NonBlock); + } + + disconnect(true); +} + // Regression test for (at least one failure mode of) // https://github.com/envoyproxy/envoy/issues/3639 where readDisable on a close // connection caused a crash. diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index 66185338b8e0..b817305aebfb 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -400,6 +400,32 @@ TEST_P(IntegrationTest, Http10WithHostandKeepAlive) { EXPECT_EQ(upstream_headers->Host()->value(), "foo.com"); } +TEST_P(IntegrationTest, Pipeline) { + autonomous_upstream_ = true; + initialize(); + std::string response; + + Buffer::OwnedImpl buffer("GET / HTTP/1.1\r\nHost: host\r\n\r\nGET / HTTP/1.1\r\n\r\n"); + RawConnectionDriver connection( + lookupPort("http"), buffer, + [&](Network::ClientConnection&, const Buffer::Instance& data) -> void { + response.append(data.toString()); + }, + version_); + // First response should be success. + while (response.find("200") == std::string::npos) { + connection.run(Event::Dispatcher::RunType::NonBlock); + } + EXPECT_THAT(response, HasSubstr("HTTP/1.1 200 OK\r\n")); + + // Second response should be 400 (no host) + while (response.find("400") == std::string::npos) { + connection.run(Event::Dispatcher::RunType::NonBlock); + } + EXPECT_THAT(response, HasSubstr("HTTP/1.1 400 Bad Request\r\n")); + connection.close(); +} + TEST_P(IntegrationTest, NoHost) { initialize(); codec_client_ = makeHttpConnection(lookupPort("http")); From e637457bfb46745bbfe4b39bd04207b00bba2064 Mon Sep 17 00:00:00 2001 From: asraa Date: Thu, 26 Sep 2019 19:19:30 -0400 Subject: [PATCH 12/32] sds: add validation callback for all secret providers (#8310) This adds an implementation of addValidationCallback for all secret providers. This is a no-op, unless the secret provider is a `CertificateValidationContextSdsApi. Envoy crashes when trying to add a validation callback to CertificateValidationContextProvider due to a bad dynamic cast. When certificate_validation_context_provider_ isn't a CertificateValidationContextSdsApi, we no-op. Fixes: #7892 and OSS-Fuzz Issues https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=17001 https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=16766 Testing: add corpus entry Signed-off-by: Asra Ali --- include/envoy/secret/secret_provider.h | 10 + source/common/secret/sds_api.h | 7 +- source/common/secret/secret_provider_impl.h | 10 + .../tls/context_config_impl.cc | 10 +- ...testcase-config_fuzz_test-5729922022113280 | 251 ++++++++++++++++++ ...inimized-config_fuzz_test-5674078337236992 | 119 +++++++++ 6 files changed, 400 insertions(+), 7 deletions(-) create mode 100644 test/server/server_corpus/clusterfuzz-testcase-config_fuzz_test-5729922022113280 create mode 100644 test/server/server_corpus/clusterfuzz-testcase-minimized-config_fuzz_test-5674078337236992 diff --git a/include/envoy/secret/secret_provider.h b/include/envoy/secret/secret_provider.h index 4ec8375d5840..c532e8e32622 100644 --- a/include/envoy/secret/secret_provider.h +++ b/include/envoy/secret/secret_provider.h @@ -23,6 +23,16 @@ template class SecretProvider { */ virtual const SecretType* secret() const PURE; + /** + * Add secret validation callback into secret provider. + * It is safe to call this method by main thread and callback is safe to be invoked + * on main thread. + * @param callback callback that is executed by secret provider. + * @return CallbackHandle the handle which can remove that validation callback. + */ + virtual Common::CallbackHandle* + addValidationCallback(std::function callback) PURE; + /** * Add secret update callback into secret provider. * It is safe to call this method by main thread and callback is safe to be invoked diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index 762de28c96b4..b9a803a068f2 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -117,6 +117,10 @@ class TlsCertificateSdsApi : public SdsApi, public TlsCertificateConfigProvider const envoy::api::v2::auth::TlsCertificate* secret() const override { return tls_certificate_secrets_.get(); } + Common::CallbackHandle* + addValidationCallback(std::function) override { + return nullptr; + } Common::CallbackHandle* addUpdateCallback(std::function callback) override { return update_callback_manager_.add(callback); } @@ -172,7 +176,8 @@ class CertificateValidationContextSdsApi : public SdsApi, } Common::CallbackHandle* addValidationCallback( - std::function callback) { + std::function callback) + override { return validation_callback_manager_.add(callback); } diff --git a/source/common/secret/secret_provider_impl.h b/source/common/secret/secret_provider_impl.h index c933a5f71625..c3f88ff37c1c 100644 --- a/source/common/secret/secret_provider_impl.h +++ b/source/common/secret/secret_provider_impl.h @@ -18,6 +18,11 @@ class TlsCertificateConfigProviderImpl : public TlsCertificateConfigProvider { return tls_certificate_.get(); } + Common::CallbackHandle* + addValidationCallback(std::function) override { + return nullptr; + } + Common::CallbackHandle* addUpdateCallback(std::function) override { return nullptr; } private: @@ -34,6 +39,11 @@ class CertificateValidationContextConfigProviderImpl return certificate_validation_context_.get(); } + Common::CallbackHandle* addValidationCallback( + std::function) override { + return nullptr; + } + Common::CallbackHandle* addUpdateCallback(std::function) override { return nullptr; } private: diff --git a/source/extensions/transport_sockets/tls/context_config_impl.cc b/source/extensions/transport_sockets/tls/context_config_impl.cc index 5978e136c880..45dee440c0de 100644 --- a/source/extensions/transport_sockets/tls/context_config_impl.cc +++ b/source/extensions/transport_sockets/tls/context_config_impl.cc @@ -133,12 +133,10 @@ ContextConfigImpl::ContextConfigImpl( // getCombinedValidationContextConfig() throws exception, validation_context_config_ will not // get updated. cvc_validation_callback_handle_ = - dynamic_cast( - certificate_validation_context_provider_.get()) - ->addValidationCallback( - [this](const envoy::api::v2::auth::CertificateValidationContext& dynamic_cvc) { - getCombinedValidationContextConfig(dynamic_cvc); - }); + certificate_validation_context_provider_->addValidationCallback( + [this](const envoy::api::v2::auth::CertificateValidationContext& dynamic_cvc) { + getCombinedValidationContextConfig(dynamic_cvc); + }); } // Load inline or static secret into tls_certificate_config_. if (!tls_certificate_providers_.empty()) { diff --git a/test/server/server_corpus/clusterfuzz-testcase-config_fuzz_test-5729922022113280 b/test/server/server_corpus/clusterfuzz-testcase-config_fuzz_test-5729922022113280 new file mode 100644 index 000000000000..a16e3180af0f --- /dev/null +++ b/test/server/server_corpus/clusterfuzz-testcase-config_fuzz_test-5729922022113280 @@ -0,0 +1,251 @@ +static_resources { + clusters { + name: "i?aress_http" + type: LOGICAL_DNS + connect_timeout { + seconds: 2304 + nanos: 707406378 + } + tls_context { + common_tls_context { + alpn_protocols: "" + alpn_protocols: "" + combined_validation_context { + default_validation_context { + trusted_ca { + inline_bytes: "\302\000\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\000\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302" + } + require_signed_certificate_timestamp { + } + } + validation_context_sds_secret_config { + } + } + } + sni: "mane" + } + alt_stat_name: "[" + } + clusters { + name: "i?aress_http" + type: LOGICAL_DNS + connect_timeout { + nanos: 538970747 + } + hosts { + pipe { + path: "\000\000\000\000`" + } + } + hosts { + pipe { + path: "2" + } + } + hosts { + pipe { + path: "`" + } + } + hosts { + pipe { + path: "2" + } + } + tls_context { + common_tls_context { + combined_validation_context { + default_validation_context { + trusted_ca { + inline_bytes: "\302\000\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\000\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302" + } + require_signed_certificate_timestamp { + } + } + validation_context_sds_secret_config { + sds_config { + api_config_source { + refresh_delay { + seconds: 8391050737688276324 + nanos: 64 + } + } + } + } + } + } + } + http_protocol_options { + allow_absolute_url { + value: true + } + } + alt_stat_name: "[" + upstream_connection_options { + tcp_keepalive { + keepalive_probes { + value: 589824 + } + } + } + } + clusters { + name: "p`ne" + type: STRICT_DNS + connect_timeout { + nanos: 707406378 + } + hosts { + pipe { + path: "2" + } + } + hosts { + pipe { + path: "`" + } + } + tls_context { + common_tls_context { + alpn_protocols: "" + alpn_protocols: "" + combined_validation_context { + default_validation_context { + trusted_ca { + inline_bytes: "\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302\302" + } + require_signed_certificate_timestamp { + } + } + validation_context_sds_secret_config { + sds_config { + api_config_source { + refresh_delay { + seconds: 8391050737688276324 + nanos: 64 + } + } + } + } + } + } + sni: "mane" + } + http_protocol_options { + allow_absolute_url { + value: true + } + } + alt_stat_name: "[" + } + secrets { + validation_context { + } + } +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { + name: "2" +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { + name: "Z" +} +stats_sinks { +} +stats_sinks { + name: "type.googleapis.com/envoy.api.v2.route.Route" + typed_config { + type_url: "type.googleapis.com/envoy.api.v2.route.Route" + value: "\nJ1" + } +} +stats_sinks { +} +stats_sinks { + typed_config { + type_url: "type.googleapis.com/envoy.api.v2.route.Route" + value: "\022*J.:*static\'_resourc\022es {(\n cluster`s" + } +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { + name: "type.googleapis.com/envoy.api.v2.route.Route" + typed_config { + type_url: "type.googleapis.com/envoy.api.v2.route.Route" + value: "type.googleapis.com/envoy.api.v2.route.Route" + } +} +stats_sinks { + name: "]" +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { + name: "type.googleatis.com/google.protobuf.Value" +} +stats_sinks { + typed_config { + type_url: "type.googleapis.com/envoy.api.v2.route.Route" + value: "\022*J.:*static_resourc\001es {(\n cluster`s" + } +} +stats_sinks { + name: "," +} +stats_sinks { +} +stats_sinks { + typed_config { + type_url: "type.googleapis.com/envoy.api.v2.route.Route" + value: "\022*J :2222622222222222222222221\022" + } +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { + name: "\013" +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { +} +stats_sinks { + typed_config { + type_url: "type.googleapis.com/envoy.api.v2.route.Route" + value: "\nJ1" + } +} +stats_sinks { +} diff --git a/test/server/server_corpus/clusterfuzz-testcase-minimized-config_fuzz_test-5674078337236992 b/test/server/server_corpus/clusterfuzz-testcase-minimized-config_fuzz_test-5674078337236992 new file mode 100644 index 000000000000..87091f058b0b --- /dev/null +++ b/test/server/server_corpus/clusterfuzz-testcase-minimized-config_fuzz_test-5674078337236992 @@ -0,0 +1,119 @@ +static_resources { + clusters { + name: "p" + connect_timeout { + nanos: 786870912 + } + hosts { + pipe { + path: "`" + } + } + hosts { + pipe { + path: "`" + } + } + tls_context { + common_tls_context { + tls_params { + ecdh_curves: "P" + ecdh_curves: "P" + } + tls_certificates { + private_key { + filename: "\001\000\000\t" + } + password { + inline_string: "#\000\000\000\000\000\000\000" + } + ocsp_staple { + filename: "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" + } + } + tls_certificates { + private_key { + filename: "\001\000\000\t" + } + password { + inline_string: "#\000\000\000\000\000\000\000" + } + ocsp_staple { + filename: "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" + } + } + tls_certificates { + private_key { + filename: "\001static_resources {\n clusters {\n name: \"z:\"\n type: STRICT_DNS\n connect_timeout {\n seconds: -210453399808\n nanos: 250002048\n }\n http2_protocol_options {\n allow_connect: true\n }\n dns_lookup_family: V4_ONLY\n common_lb_config {\n ignore_new_hosts_until_first_hc: true\n }\n alt_stat_name: \"\\020\"\n close_connections_on_host_health_failure: true\n filters {\n name: \"%\"\n }\n }\n clusters {\n name: \"service_google\"\n connect_timeout {\n nanos\000: 786870912\n }\n per_connection_buffer_limit_bytes {\n value: 4143972352\n }\n lb_policy: RING_HASH\n hosts {\n pipe {\n path: \"`\"\n }\n }\n hosts {\n pipe {\n path: \"#\"\n }\n }\n hosts {\n pipe {\n path: \"`\"\n }\n }\n tls_context {\n common_tls_conte\000xt {\n tls_params {\n ecdh_curves: \"P\"\n }\n tls_certificates {\n private_key {\n filename: \"\\001\\0\t00\\000\\t\"\n }\n password {\n inline_string: \"#\\000\\000\\000\\000\\000\\000\\000\"\n }\n ocsp_staple {\n filename: \"\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\\000\"\n }\n }\n tls_certificates" + } + ocsp_staple { + filename: "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" + } + } + combined_validation_context { + default_validation_context { + verify_subject_alt_name: "\001\000\000\000\000\000\000\006" + require_ocsp_staple { + value: true + } + require_signed_certificate_timestamp { + } + } + validation_context_sds_secret_config { + } + } + } + } + close_connections_on_host_health_failure: true + } + secrets { + validation_context { + trusted_ca { + inline_bytes: "\316" + } + } + } +} +cluster_manager { + local_cluster_name: "," + upstream_bind_config { + source_address { + address: "\032" + port_value: 8192 + ipv4_compat: true + } + } + load_stats_config { + refresh_delay { + seconds: 4611686018427387903 + } + set_node_on_first_message_only: true + } +} +stats_sinks { + typed_config { + type_url: "\001\000\000\t" + value: "\230*Vs{{{{{{{{{{{{{{{{{{{{{{{{{{||{|||{{{{{{{{{{{{{{{{{{{{{{{{{||||||||||||||||||||||||||st|||||||||||||||||||||stt" + } +} +stats_sinks { +} +stats_sinks { + name: "VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" + typed_config { + type_url: "\001\000\000\t" + } +} +watchdog { + miss_timeout { + seconds: 2147483707 + } + kill_timeout { + seconds: 2046820352 + } +} +hds_config { +} +layered_runtime { +} +header_prefix: ":" From 153bf8251be76e5eba8b04f2aeeb5d7a3a490690 Mon Sep 17 00:00:00 2001 From: Venil Noronha Date: Fri, 27 Sep 2019 08:25:09 -0700 Subject: [PATCH 13/32] dns: enable dns failure refresh rate configuration (#8226) This adds a new dns_failure_refresh_rate configuration to the Cluster type to allow the configuration of the DNS refresh rate during failures (empty responses). Risk Level: Low Testing: Updated tests Docs Changes: Updated docs Release Notes: Added an entry describing the change Fixes #7367 Signed-off-by: Venil Noronha --- api/envoy/api/v2/cds.proto | 31 +++++++++++++++++- api/envoy/api/v3alpha/cds.proto | 31 +++++++++++++++++- .../upstream/service_discovery.rst | 6 ++-- docs/root/intro/version_history.rst | 1 + source/common/common/backoff_strategy.cc | 6 ++++ source/common/common/backoff_strategy.h | 25 +++++++++++++-- source/common/config/BUILD | 1 + source/common/config/utility.cc | 17 ++++++++++ source/common/config/utility.h | 12 +++++++ source/common/upstream/logical_dns_cluster.cc | 7 ++++ source/common/upstream/logical_dns_cluster.h | 1 + source/common/upstream/strict_dns_cluster.cc | 23 +++++++++---- source/common/upstream/strict_dns_cluster.h | 1 + test/common/common/backoff_strategy_test.cc | 10 ++++++ test/common/config/utility_test.cc | 31 ++++++++++++++++++ .../upstream/logical_dns_cluster_test.cc | 10 +++++- test/common/upstream/upstream_impl_test.cc | 32 +++++++++++++++++-- test/integration/stats_integration_test.cc | 6 ++-- 18 files changed, 234 insertions(+), 17 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index 719fe0f6ab91..bd030b90fc79 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -47,7 +47,7 @@ service ClusterDiscoveryService { } // Configuration for a single upstream cluster. -// [#comment:next free field: 44] +// [#comment:next free field: 45] message Cluster { // Refer to :ref:`service discovery type ` // for an explanation on each type. @@ -436,6 +436,22 @@ message Cluster { bool close_connections_on_host_set_change = 6; } + message RefreshRate { + // Specifies the base interval between refreshes. This parameter is required and must be greater + // than zero and less than + // :ref:`max_interval `. + google.protobuf.Duration base_interval = 1 [(validate.rules).duration = { + required: true + gt {nanos: 1000000} + }]; + + // Specifies the maximum interval between refreshes. This parameter is optional, but must be + // greater than or equal to the + // :ref:`base_interval ` if set. The default + // is 10 times the :ref:`base_interval `. + google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}]; + } + reserved 12, 15; // Configuration to use different transport sockets for different endpoints. @@ -614,6 +630,19 @@ message Cluster { // this setting is ignored. google.protobuf.Duration dns_refresh_rate = 16 [(validate.rules).duration = {gt {}}]; + // If the DNS failure refresh rate is specified and the cluster type is either + // :ref:`STRICT_DNS`, + // or :ref:`LOGICAL_DNS`, + // this is used as the cluster’s DNS refresh rate when requests are failing. If this setting is + // not specified, the failure refresh rate defaults to the DNS refresh rate. For cluster types + // other than :ref:`STRICT_DNS` and + // :ref:`LOGICAL_DNS` this setting is + // ignored. + // + // Note: Currently, DNS failures and empty DNS responses are not treated differently and this + // configuration is applied in both situations. + RefreshRate dns_failure_refresh_rate = 44; + // Optional configuration for setting cluster's DNS refresh rate. If the value is set to true, // cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS // resolution. diff --git a/api/envoy/api/v3alpha/cds.proto b/api/envoy/api/v3alpha/cds.proto index 636cf46fa6d4..eae8b5786742 100644 --- a/api/envoy/api/v3alpha/cds.proto +++ b/api/envoy/api/v3alpha/cds.proto @@ -47,7 +47,7 @@ service ClusterDiscoveryService { } // Configuration for a single upstream cluster. -// [#comment:next free field: 44] +// [#comment:next free field: 45] message Cluster { // Refer to :ref:`service discovery type ` // for an explanation on each type. @@ -430,6 +430,22 @@ message Cluster { bool close_connections_on_host_set_change = 6; } + message RefreshRate { + // Specifies the base interval between refreshes. This parameter is required and must be greater + // than zero and less than + // :ref:`max_interval `. + google.protobuf.Duration base_interval = 1 [(validate.rules).duration = { + required: true + gt {nanos: 1000000} + }]; + + // Specifies the maximum interval between refreshes. This parameter is optional, but must be + // greater than or equal to the + // :ref:`base_interval ` if set. The default + // is 10 times the :ref:`base_interval `. + google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}]; + } + reserved 12, 15; // Configuration to use different transport sockets for different endpoints. @@ -608,6 +624,19 @@ message Cluster { // this setting is ignored. google.protobuf.Duration dns_refresh_rate = 16 [(validate.rules).duration = {gt {}}]; + // If the DNS failure refresh rate is specified and the cluster type is either + // :ref:`STRICT_DNS`, + // or :ref:`LOGICAL_DNS`, + // this is used as the cluster’s DNS refresh rate when requests are failing. If this setting is + // not specified, the failure refresh rate defaults to the DNS refresh rate. For cluster types + // other than :ref:`STRICT_DNS` and + // :ref:`LOGICAL_DNS` this setting is + // ignored. + // + // Note: Currently, DNS failures and empty DNS responses are not treated differently and this + // configuration is applied in both situations. + RefreshRate dns_failure_refresh_rate = 44; + // Optional configuration for setting cluster's DNS refresh rate. If the value is set to true, // cluster's DNS refresh rate will be set to resource record's TTL which comes from DNS // resolution. diff --git a/docs/root/intro/arch_overview/upstream/service_discovery.rst b/docs/root/intro/arch_overview/upstream/service_discovery.rst index 3f6bd5d7b4f4..5147d3ef162c 100644 --- a/docs/root/intro/arch_overview/upstream/service_discovery.rst +++ b/docs/root/intro/arch_overview/upstream/service_discovery.rst @@ -43,7 +43,8 @@ If :ref:`respect_dns_ttl ` is enabled, :ref:`dns_refresh_rate ` are used to control DNS refresh rate. For strict DNS cluster, if the minimum of all record TTLs is 0, :ref:`dns_refresh_rate ` will be used as the cluster's DNS refresh rate. :ref:`dns_refresh_rate ` -defaults to 5000ms if not specified. +defaults to 5000ms if not specified. The :ref:`dns_failure_refresh_rate ` +controls the refresh frequency during failures, and, if not configured, the DNS refresh rate will be used. .. _arch_overview_service_discovery_types_logical_dns: @@ -68,7 +69,8 @@ If :ref:`respect_dns_ttl ` is enabled, :ref:`dns_refresh_rate ` are used to control DNS refresh rate. For logical DNS cluster, if the TTL of first record is 0, :ref:`dns_refresh_rate ` will be used as the cluster's DNS refresh rate. :ref:`dns_refresh_rate ` -defaults to 5000ms if not specified. +defaults to 5000ms if not specified. The :ref:`dns_failure_refresh_rate ` +controls the refresh frequency during failures, and, if not configured, the DNS refresh rate will be used. .. _arch_overview_service_discovery_types_original_destination: diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 969be0791e53..be5176d560d2 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -22,6 +22,7 @@ Version history * config: async data access for local and remote data source. * config: changed the default value of :ref:`initial_fetch_timeout ` from 0s to 15s. This is a change in behaviour in the sense that Envoy will move to the next initialization phase, even if the first config is not delivered in 15s. Refer to :ref:`initialization process ` for more details. * config: added stat :ref:`init_fetch_timeout `. +* dns: added support for configuring :ref:`dns_failure_refresh_rate ` to set the DNS refresh rate during failures. * ext_authz: added :ref:`configurable ability ` to send dynamic metadata to the `ext_authz` service. * ext_authz: added tracing to the HTTP client. * fault: added overrides for default runtime keys in :ref:`HTTPFault ` filter. diff --git a/source/common/common/backoff_strategy.cc b/source/common/common/backoff_strategy.cc index a67cc3cb88d1..8431b36b73bb 100644 --- a/source/common/common/backoff_strategy.cc +++ b/source/common/common/backoff_strategy.cc @@ -21,4 +21,10 @@ uint64_t JitteredBackOffStrategy::nextBackOffMs() { void JitteredBackOffStrategy::reset() { current_retry_ = 1; } +FixedBackOffStrategy::FixedBackOffStrategy(uint64_t interval_ms) : interval_ms_(interval_ms) { + ASSERT(interval_ms_ > 0); +} + +uint64_t FixedBackOffStrategy::nextBackOffMs() { return interval_ms_; } + } // namespace Envoy diff --git a/source/common/common/backoff_strategy.h b/source/common/common/backoff_strategy.h index 708ac3681cda..ed6d9955f09f 100644 --- a/source/common/common/backoff_strategy.h +++ b/source/common/common/backoff_strategy.h @@ -18,10 +18,10 @@ class JitteredBackOffStrategy : public BackOffStrategy { public: /** * Constructs fully jittered backoff strategy. - * @param base_interval the base_interval to be used for next backoff computation. It should be + * @param base_interval the base interval to be used for next backoff computation. It should be * greater than zero and less than or equal to max_interval. * @param max_interval the cap on the next backoff value. - * @param random the random generator + * @param random the random generator. */ JitteredBackOffStrategy(uint64_t base_interval, uint64_t max_interval, Runtime::RandomGenerator& random); @@ -36,4 +36,25 @@ class JitteredBackOffStrategy : public BackOffStrategy { uint64_t current_retry_{1}; Runtime::RandomGenerator& random_; }; + +/** + * Implementation of BackOffStrategy that uses a fixed backoff. + */ +class FixedBackOffStrategy : public BackOffStrategy { + +public: + /** + * Constructs fixed backoff strategy. + * @param interval_ms the fixed backoff duration. It should be greater than zero. + */ + FixedBackOffStrategy(uint64_t interval_ms); + + // BackOffStrategy methods. + uint64_t nextBackOffMs() override; + void reset() override {} + +private: + const uint64_t interval_ms_; +}; + } // namespace Envoy diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 6d82c8e0a3f6..5d346543180c 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -377,6 +377,7 @@ envoy_cc_library( "//include/envoy/server:filter_config_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/common:assert_lib", + "//source/common/common:backoff_lib", "//source/common/common:hash_lib", "//source/common/common:hex_lib", "//source/common/grpc:common_lib", diff --git a/source/common/config/utility.cc b/source/common/config/utility.cc index 84dedbcda38f..48f7d4892a1e 100644 --- a/source/common/config/utility.cc +++ b/source/common/config/utility.cc @@ -308,5 +308,22 @@ bool Utility::allowDeprecatedV1Config(Runtime::Loader& runtime, const Json::Obje return true; } +BackOffStrategyPtr Utility::prepareDnsRefreshStrategy(const envoy::api::v2::Cluster& cluster, + const uint64_t dns_refresh_rate_ms, + Runtime::RandomGenerator& random) { + if (cluster.has_dns_failure_refresh_rate()) { + uint64_t base_interval_ms = + PROTOBUF_GET_MS_REQUIRED(cluster.dns_failure_refresh_rate(), base_interval); + uint64_t max_interval_ms = PROTOBUF_GET_MS_OR_DEFAULT(cluster.dns_failure_refresh_rate(), + max_interval, base_interval_ms * 10); + if (max_interval_ms < base_interval_ms) { + throw EnvoyException("cluster.dns_failure_refresh_rate must have max_interval greater than " + "or equal to the base_interval"); + } + return std::make_unique(base_interval_ms, max_interval_ms, random); + } + return std::make_unique(dns_refresh_rate_ms); +} + } // namespace Config } // namespace Envoy diff --git a/source/common/config/utility.h b/source/common/config/utility.h index 935699ecbdba..0fecec36b045 100644 --- a/source/common/config/utility.h +++ b/source/common/config/utility.h @@ -16,6 +16,7 @@ #include "envoy/upstream/cluster_manager.h" #include "common/common/assert.h" +#include "common/common/backoff_strategy.h" #include "common/common/hash.h" #include "common/common/hex.h" #include "common/grpc/common.h" @@ -311,6 +312,17 @@ class Utility { name, filter_type)); } } + + /** + * Prepares the DNS failure refresh backoff strategy given the cluster configuration. + * @param cluster the cluster configuration. + * @param dns_refresh_rate_ms the default DNS refresh rate. + * @param random the random generator. + * @return BackOffStrategyPtr for scheduling refreshes. + */ + static BackOffStrategyPtr prepareDnsRefreshStrategy(const envoy::api::v2::Cluster& cluster, + uint64_t dns_refresh_rate_ms, + Runtime::RandomGenerator& random); }; } // namespace Config diff --git a/source/common/upstream/logical_dns_cluster.cc b/source/common/upstream/logical_dns_cluster.cc index 06c1e415b845..f9096ee19426 100644 --- a/source/common/upstream/logical_dns_cluster.cc +++ b/source/common/upstream/logical_dns_cluster.cc @@ -33,6 +33,9 @@ LogicalDnsCluster::LogicalDnsCluster( load_assignment_(cluster.has_load_assignment() ? cluster.load_assignment() : Config::Utility::translateClusterHosts(cluster.hosts())) { + failure_backoff_strategy_ = Config::Utility::prepareDnsRefreshStrategy( + cluster, dns_refresh_rate_ms_.count(), factory_context.random()); + const auto& locality_lb_endpoints = load_assignment_.endpoints(); if (locality_lb_endpoints.size() != 1 || locality_lb_endpoints[0].lb_endpoints().size() != 1) { if (cluster.has_load_assignment()) { @@ -110,6 +113,10 @@ void LogicalDnsCluster::startResolve() { // checking, and creating real host connections. logical_host_->setNewAddress(new_address, lbEndpoint()); } + + failure_backoff_strategy_->reset(); + } else { + refresh_rate = std::chrono::milliseconds(failure_backoff_strategy_->nextBackOffMs()); } onPreInitComplete(); diff --git a/source/common/upstream/logical_dns_cluster.h b/source/common/upstream/logical_dns_cluster.h index 68e786c80af6..b1f7026aa312 100644 --- a/source/common/upstream/logical_dns_cluster.h +++ b/source/common/upstream/logical_dns_cluster.h @@ -62,6 +62,7 @@ class LogicalDnsCluster : public ClusterImplBase { Network::DnsResolverSharedPtr dns_resolver_; const std::chrono::milliseconds dns_refresh_rate_ms_; + BackOffStrategyPtr failure_backoff_strategy_; const bool respect_dns_ttl_; Network::DnsLookupFamily dns_lookup_family_; Event::TimerPtr resolve_timer_; diff --git a/source/common/upstream/strict_dns_cluster.cc b/source/common/upstream/strict_dns_cluster.cc index d1f4ef3039f1..49cdd1d50885 100644 --- a/source/common/upstream/strict_dns_cluster.cc +++ b/source/common/upstream/strict_dns_cluster.cc @@ -14,6 +14,9 @@ StrictDnsClusterImpl::StrictDnsClusterImpl( dns_refresh_rate_ms_( std::chrono::milliseconds(PROTOBUF_GET_MS_OR_DEFAULT(cluster, dns_refresh_rate, 5000))), respect_dns_ttl_(cluster.respect_dns_ttl()) { + failure_backoff_strategy_ = Config::Utility::prepareDnsRefreshStrategy( + cluster, dns_refresh_rate_ms_.count(), factory_context.random()); + std::list resolve_targets; const envoy::api::v2::ClusterLoadAssignment load_assignment( cluster.has_load_assignment() ? cluster.load_assignment() @@ -138,12 +141,20 @@ void StrictDnsClusterImpl::ResolveTarget::startResolve() { std::chrono::milliseconds final_refresh_rate = parent_.dns_refresh_rate_ms_; - if (parent_.respect_dns_ttl_ && ttl_refresh_rate != std::chrono::seconds(0) && - !response.empty()) { - final_refresh_rate = ttl_refresh_rate; - ASSERT(ttl_refresh_rate != std::chrono::seconds::max() && final_refresh_rate.count() > 0); - ENVOY_LOG(debug, "DNS refresh rate reset for {}, refresh rate {} ms", dns_address_, - final_refresh_rate.count()); + if (response.empty()) { + final_refresh_rate = + std::chrono::milliseconds(parent_.failure_backoff_strategy_->nextBackOffMs()); + ENVOY_LOG(debug, "DNS refresh rate reset for {}, (failure) refresh rate {} ms", + dns_address_, final_refresh_rate.count()); + } else { + parent_.failure_backoff_strategy_->reset(); + if (parent_.respect_dns_ttl_ && ttl_refresh_rate != std::chrono::seconds(0)) { + final_refresh_rate = ttl_refresh_rate; + ASSERT(ttl_refresh_rate != std::chrono::seconds::max() && + final_refresh_rate.count() > 0); + ENVOY_LOG(debug, "DNS refresh rate reset for {}, refresh rate {} ms", dns_address_, + final_refresh_rate.count()); + } } resolve_timer_->enableTimer(final_refresh_rate); diff --git a/source/common/upstream/strict_dns_cluster.h b/source/common/upstream/strict_dns_cluster.h index 2b0c8f4f135d..9dab1adb631d 100644 --- a/source/common/upstream/strict_dns_cluster.h +++ b/source/common/upstream/strict_dns_cluster.h @@ -52,6 +52,7 @@ class StrictDnsClusterImpl : public BaseDynamicClusterImpl { Network::DnsResolverSharedPtr dns_resolver_; std::list resolve_targets_; const std::chrono::milliseconds dns_refresh_rate_ms_; + BackOffStrategyPtr failure_backoff_strategy_; const bool respect_dns_ttl_; Network::DnsLookupFamily dns_lookup_family_; uint32_t overprovisioning_factor_; diff --git a/test/common/common/backoff_strategy_test.cc b/test/common/common/backoff_strategy_test.cc index 010768349983..9d7bc866ed87 100644 --- a/test/common/common/backoff_strategy_test.cc +++ b/test/common/common/backoff_strategy_test.cc @@ -56,4 +56,14 @@ TEST(BackOffStrategyTest, JitteredBackOffWithMaxIntervalReset) { jittered_back_off.reset(); EXPECT_EQ(4, jittered_back_off.nextBackOffMs()); // Should start from start } + +TEST(BackOffStrategyTest, FixedBackOffBasicReset) { + FixedBackOffStrategy fixed_back_off(30); + EXPECT_EQ(30, fixed_back_off.nextBackOffMs()); + EXPECT_EQ(30, fixed_back_off.nextBackOffMs()); + + fixed_back_off.reset(); + EXPECT_EQ(30, fixed_back_off.nextBackOffMs()); +} + } // namespace Envoy diff --git a/test/common/config/utility_test.cc b/test/common/config/utility_test.cc index a84513f03633..80891523e276 100644 --- a/test/common/config/utility_test.cc +++ b/test/common/config/utility_test.cc @@ -264,6 +264,37 @@ TEST(UtilityTest, FactoryForGrpcApiConfigSource) { } } +TEST(UtilityTest, PrepareDnsRefreshStrategy) { + NiceMock random; + + { + // dns_failure_refresh_rate not set. + envoy::api::v2::Cluster cluster; + BackOffStrategyPtr strategy = Utility::prepareDnsRefreshStrategy(cluster, 5000, random); + EXPECT_NE(nullptr, dynamic_cast(strategy.get())); + } + + { + // dns_failure_refresh_rate set. + envoy::api::v2::Cluster cluster; + cluster.mutable_dns_failure_refresh_rate()->mutable_base_interval()->set_seconds(7); + cluster.mutable_dns_failure_refresh_rate()->mutable_max_interval()->set_seconds(10); + BackOffStrategyPtr strategy = Utility::prepareDnsRefreshStrategy(cluster, 5000, random); + EXPECT_NE(nullptr, dynamic_cast(strategy.get())); + } + + { + // dns_failure_refresh_rate set with invalid max_interval. + envoy::api::v2::Cluster cluster; + cluster.mutable_dns_failure_refresh_rate()->mutable_base_interval()->set_seconds(7); + cluster.mutable_dns_failure_refresh_rate()->mutable_max_interval()->set_seconds(2); + EXPECT_THROW_WITH_REGEX(Utility::prepareDnsRefreshStrategy(cluster, 5000, random), + EnvoyException, + "cluster.dns_failure_refresh_rate must have max_interval greater than " + "or equal to the base_interval"); + } +} + TEST(CheckApiConfigSourceSubscriptionBackingClusterTest, GrpcClusterTestAcrossTypes) { envoy::api::v2::core::ConfigSource config; auto* api_config_source = config.mutable_api_config_source(); diff --git a/test/common/upstream/logical_dns_cluster_test.cc b/test/common/upstream/logical_dns_cluster_test.cc index b3d36e7c43b4..784d994efdae 100644 --- a/test/common/upstream/logical_dns_cluster_test.cc +++ b/test/common/upstream/logical_dns_cluster_test.cc @@ -30,6 +30,7 @@ using testing::_; using testing::Invoke; using testing::NiceMock; +using testing::Return; namespace Envoy { namespace Upstream { @@ -154,7 +155,8 @@ class LogicalDnsClusterTest : public testing::Test { resolve_timer_->invokeCallback(); // Empty should not cause any change. - EXPECT_CALL(*resolve_timer_, enableTimer(_, _)); + ON_CALL(random_, random()).WillByDefault(Return(6000)); + EXPECT_CALL(*resolve_timer_, enableTimer(std::chrono::milliseconds(6000), _)); dns_callback_({}); EXPECT_EQ(logical_host, cluster_->prioritySet().hostSetsPerPriority()[0]->hosts()[0]); @@ -381,6 +383,9 @@ TEST_F(LogicalDnsClusterTest, Basic) { name: name type: LOGICAL_DNS dns_refresh_rate: 4s + dns_failure_refresh_rate: + base_interval: 7s + max_interval: 10s connect_timeout: 0.25s lb_policy: ROUND_ROBIN # Since the following expectResolve() requires Network::DnsLookupFamily::V4Only we need to set @@ -396,6 +401,9 @@ TEST_F(LogicalDnsClusterTest, Basic) { name: name type: LOGICAL_DNS dns_refresh_rate: 4s + dns_failure_refresh_rate: + base_interval: 7s + max_interval: 10s connect_timeout: 0.25s lb_policy: ROUND_ROBIN # Since the following expectResolve() requires Network::DnsLookupFamily::V4Only we need to set diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 9934f68f7ceb..40a923e304b8 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -39,6 +39,7 @@ using testing::_; using testing::ContainerEq; using testing::Invoke; using testing::NiceMock; +using testing::Return; namespace Envoy { namespace Upstream { @@ -235,6 +236,9 @@ TEST_F(StrictDnsClusterImplTest, Basic) { connect_timeout: 0.25s type: strict_dns dns_refresh_rate: 4s + dns_failure_refresh_rate: + base_interval: 7s + max_interval: 10s lb_policy: round_robin circuit_breakers: thresholds: @@ -349,6 +353,26 @@ TEST_F(StrictDnsClusterImplTest, Basic) { EXPECT_EQ(cluster.info().get(), &host->cluster()); } + // Empty response. + resolver1.expectResolve(*dns_resolver_); + resolver1.timer_->invokeCallback(); + ON_CALL(random_, random()).WillByDefault(Return(8000)); + EXPECT_CALL(*resolver1.timer_, enableTimer(std::chrono::milliseconds(1000), _)); + EXPECT_CALL(membership_updated, ready()); + resolver1.dns_callback_(TestUtility::makeDnsResponse({})); + EXPECT_THAT( + std::list({"10.0.0.1:11002"}), + ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); + + resolver2.expectResolve(*dns_resolver_); + resolver2.timer_->invokeCallback(); + EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(8000), _)); + EXPECT_CALL(membership_updated, ready()); + resolver2.dns_callback_(TestUtility::makeDnsResponse({})); + EXPECT_THAT( + std::list({}), + ContainerEq(hostListToAddresses(cluster.prioritySet().hostSetsPerPriority()[0]->hosts()))); + // Make sure we cancel. resolver1.expectResolve(*dns_resolver_); resolver1.timer_->invokeCallback(); @@ -496,6 +520,9 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { dns_lookup_family: V4_ONLY connect_timeout: 0.25s dns_refresh_rate: 4s + dns_failure_refresh_rate: + base_interval: 7s + max_interval: 10s lb_policy: ROUND_ROBIN @@ -702,11 +729,12 @@ TEST_F(StrictDnsClusterImplTest, LoadAssignmentBasic) { } }); - EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(4000), _)); + ON_CALL(random_, random()).WillByDefault(Return(8000)); + EXPECT_CALL(*resolver2.timer_, enableTimer(std::chrono::milliseconds(1000), _)); EXPECT_CALL(membership_updated, ready()); resolver2.dns_callback_(TestUtility::makeDnsResponse({})); - EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(4000), _)); + EXPECT_CALL(*resolver3.timer_, enableTimer(std::chrono::milliseconds(8000), _)); EXPECT_CALL(membership_updated, ready()); resolver3.dns_callback_(TestUtility::makeDnsResponse({})); diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index f7ffde2476ea..3f3d93df1412 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -253,6 +253,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // 2019/08/13 7877 42838 44000 skip EdfScheduler creation if all host weights equal // 2019/09/02 8118 42830 43000 Share symbol-tables in cluster/host stats. // 2019/09/16 8100 42894 43000 Add transport socket matcher in cluster. + // 2019/09/25 8226 43022 44000 dns: enable dns failure refresh rate configuration // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -262,7 +263,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithFakeSymbolTable) { // On a local clang8/libstdc++/linux flow, the memory usage was observed in // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may // vary. - EXPECT_MEMORY_EQ(m_per_cluster, 42894); // 104 bytes higher than a debug build. + EXPECT_MEMORY_EQ(m_per_cluster, 43022); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 44000); } @@ -289,6 +290,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // 2019/08/09 7882 35489 36000 Initial version // 2019/09/02 8118 34585 34500 Share symbol-tables in cluster/host stats. // 2019/09/16 8100 34585 34500 Add transport socket matcher in cluster. + // 2019/09/25 8226 34777 35000 dns: enable dns failure refresh rate configuration // Note: when adjusting this value: EXPECT_MEMORY_EQ is active only in CI // 'release' builds, where we control the platform and tool-chain. So you @@ -298,7 +300,7 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithRealSymbolTable) { // On a local clang8/libstdc++/linux flow, the memory usage was observed in // June 2019 to be 64 bytes higher than it is in CI/release. Your mileage may // vary. - EXPECT_MEMORY_EQ(m_per_cluster, 34649); // 104 bytes higher than a debug build. + EXPECT_MEMORY_EQ(m_per_cluster, 34777); // 104 bytes higher than a debug build. EXPECT_MEMORY_LE(m_per_cluster, 36000); } From 10db4a05f6aa0bf0adb368f4501fa7f262d28eba Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Fri, 27 Sep 2019 08:28:52 -0700 Subject: [PATCH 14/32] build: make compilation database script RBE compatible (#8389) Also moves to python3. Not moving clang-tidy to AZP yet to wait AZP stabilize a bit more. Risk Level: Low Testing: local, CI Signed-off-by: Lizan Zhou --- bazel/gen_compilation_database.sh | 9 ------- bazel/repositories.bzl | 1 + bazel/repository_locations.bzl | 5 ++++ ci/run_clang_tidy.sh | 4 --- tools/gen_compilation_database.py | 43 ++++++++++++++++++++----------- 5 files changed, 34 insertions(+), 28 deletions(-) delete mode 100755 bazel/gen_compilation_database.sh diff --git a/bazel/gen_compilation_database.sh b/bazel/gen_compilation_database.sh deleted file mode 100755 index d642ddfd285d..000000000000 --- a/bazel/gen_compilation_database.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -RELEASE_VERSION=0.3.5 - -if [[ ! -d bazel-compilation-database-${RELEASE_VERSION} ]]; then - curl -L https://github.com/grailbio/bazel-compilation-database/archive/${RELEASE_VERSION}.tar.gz | tar -xz -fi - -bazel-compilation-database-${RELEASE_VERSION}/generate.sh $@ diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 95e59466b58c..311048a98f90 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -157,6 +157,7 @@ def envoy_dependencies(skip_targets = []): _repository_impl("com_googlesource_code_re2") _com_google_cel_cpp() _repository_impl("bazel_toolchains") + _repository_impl("bazel_compdb") _repository_impl("envoy_build_tools") _python_deps() diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 9abaf46a9ecf..62c5a95d3ec0 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -1,4 +1,9 @@ REPOSITORY_LOCATIONS = dict( + bazel_compdb = dict( + sha256 = "bb1b812396e2ee36a50a13b03ae6833173ce643e8a4bd50731067d0b4e5c6e86", + strip_prefix = "bazel-compilation-database-0.3.5", + urls = ["https://github.com/grailbio/bazel-compilation-database/archive/0.3.5.tar.gz"], + ), bazel_gazelle = dict( sha256 = "be9296bfd64882e3c08e3283c58fcb461fa6dd3c171764fcc4cf322f60615a9b", urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/0.18.1/bazel-gazelle-0.18.1.tar.gz"], diff --git a/ci/run_clang_tidy.sh b/ci/run_clang_tidy.sh index 75e7146727db..924fcb2db6a0 100755 --- a/ci/run_clang_tidy.sh +++ b/ci/run_clang_tidy.sh @@ -18,10 +18,6 @@ function cleanup() { } trap cleanup EXIT -# The compilation database generate script doesn't support passing build options via CLI. -# Writing them into bazelrc -echo "build ${BAZEL_BUILD_OPTIONS}" >> .bazelrc - # bazel build need to be run to setup virtual includes, generating files which are consumed # by clang-tidy "${ENVOY_SRCDIR}/tools/gen_compilation_database.py" --run_bazel_build --include_headers diff --git a/tools/gen_compilation_database.py b/tools/gen_compilation_database.py index b987a403111f..12401e5fff14 100755 --- a/tools/gen_compilation_database.py +++ b/tools/gen_compilation_database.py @@ -1,20 +1,39 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import argparse +import glob import json import os +import shlex import subprocess +from pathlib import Path +# This method is equivalent to https://github.com/grailbio/bazel-compilation-database/blob/master/generate.sh def generateCompilationDatabase(args): + # We need to download all remote outputs for generated source code, we don't care about built + # binaries so just always strip and use dynamic link to minimize download size. + bazel_options = shlex.split(os.environ.get("BAZEL_BUILD_OPTIONS", "")) + [ + "-c", "fastbuild", "--build_tag_filters=-manual", + "--experimental_remote_download_outputs=all", "--strip=always", + "--define=dynamic_link_tests=true" + ] if args.run_bazel_build: - subprocess.check_call( - ["bazel", "build", "--build_tag_filters=-manual", "--jobs=" + os.environ.get('NUM_CPUS')] + - args.bazel_targets) + subprocess.check_call(["bazel", "build"] + bazel_options + args.bazel_targets) - gen_compilation_database_sh = os.path.join(os.path.realpath(os.path.dirname(__file__)), - "../bazel/gen_compilation_database.sh") - subprocess.check_call([gen_compilation_database_sh] + args.bazel_targets) + subprocess.check_call(["bazel", "build"] + bazel_options + [ + "--aspects=@bazel_compdb//:aspects.bzl%compilation_database_aspect", + "--output_groups=compdb_files" + ] + args.bazel_targets) + + execroot = subprocess.check_output(["bazel", "info", "execution_root"] + + bazel_options).decode().strip() + + compdb = [] + for compdb_file in Path(execroot).glob("**/*.compile_commands.json"): + compdb.extend(json.loads("[" + compdb_file.read_text().replace("__EXEC_ROOT__", execroot) + + "]")) + return compdb def isHeader(filename): @@ -61,14 +80,9 @@ def modifyCompileCommand(target, args): return target -def fixCompilationDatabase(args): - with open("compile_commands.json", "r") as db_file: - db = json.load(db_file) - +def fixCompilationDatabase(args, db): db = [modifyCompileCommand(target, args) for target in db if isCompileTarget(target, args)] - # Remove to avoid writing into symlink - os.remove("compile_commands.json") with open("compile_commands.json", "w") as db_file: json.dump(db, db_file, indent=2) @@ -84,5 +98,4 @@ def fixCompilationDatabase(args): nargs='*', default=["//source/...", "//test/...", "//tools/..."]) args = parser.parse_args() - generateCompilationDatabase(args) - fixCompilationDatabase(args) + fixCompilationDatabase(args, generateCompilationDatabase(args)) From 555f88bff95185f64939382fc768cd462c1849af Mon Sep 17 00:00:00 2001 From: Wayne Zhang Date: Fri, 27 Sep 2019 09:30:32 -0600 Subject: [PATCH 15/32] Revert "use envoy_google_grpc_external_deps for google_grpc_context_lib (#8380)" (#8407) This reverts commit 6d772458b1250c9f067f008a1a3c3dcfc1ce2a15. Signed-off-by: Wayne Zhang --- source/common/grpc/BUILD | 4 ++-- source/common/grpc/google_grpc_context.cc | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/source/common/grpc/BUILD b/source/common/grpc/BUILD index c140f561e31d..f4d6d906e70c 100644 --- a/source/common/grpc/BUILD +++ b/source/common/grpc/BUILD @@ -3,7 +3,6 @@ licenses(["notice"]) # Apache 2 load( "//bazel:envoy_build_system.bzl", "envoy_cc_library", - "envoy_google_grpc_external_deps", "envoy_package", "envoy_select_google_grpc", ) @@ -163,12 +162,13 @@ envoy_cc_library( name = "google_grpc_context_lib", srcs = ["google_grpc_context.cc"], hdrs = ["google_grpc_context.h"], + external_deps = ["grpc"], deps = [ "//source/common/common:assert_lib", "//source/common/common:lock_guard_lib", "//source/common/common:macros", "//source/common/common:thread_lib", - ] + envoy_google_grpc_external_deps(), + ], ) envoy_cc_library( diff --git a/source/common/grpc/google_grpc_context.cc b/source/common/grpc/google_grpc_context.cc index a8ad82979ec6..b2b27ce8065a 100644 --- a/source/common/grpc/google_grpc_context.cc +++ b/source/common/grpc/google_grpc_context.cc @@ -7,9 +7,7 @@ #include "common/common/macros.h" #include "common/common/thread.h" -#ifdef ENVOY_GOOGLE_GRPC #include "grpcpp/grpcpp.h" -#endif namespace Envoy { namespace Grpc { @@ -17,9 +15,7 @@ namespace Grpc { GoogleGrpcContext::GoogleGrpcContext() : instance_tracker_(instanceTracker()) { Thread::LockGuard lock(instance_tracker_.mutex_); if (++instance_tracker_.live_instances_ == 1) { -#ifdef ENVOY_GOOGLE_GRPC grpc_init(); -#endif } } @@ -32,9 +28,7 @@ GoogleGrpcContext::~GoogleGrpcContext() { Thread::LockGuard lock(instance_tracker_.mutex_); ASSERT(instance_tracker_.live_instances_ > 0); if (--instance_tracker_.live_instances_ == 0) { -#ifdef ENVOY_GOOGLE_GRPC grpc_shutdown_blocking(); // Waiting for quiescence avoids non-determinism in tests. -#endif } } From c25b5e04ce04344d938bf8d0c3b09e0f14c24dac Mon Sep 17 00:00:00 2001 From: Tony Allen Date: Fri, 27 Sep 2019 09:01:28 -0700 Subject: [PATCH 16/32] adaptive concurrency: Add configurable jitter to minRTT calculation windows (#8377) Signed-off-by: Tony Allen --- .../v2alpha/adaptive_concurrency.proto | 7 + .../gradient_controller.cc | 24 ++- .../gradient_controller.h | 11 +- .../gradient_controller_test.cc | 158 +++++++++++------- 4 files changed, 136 insertions(+), 64 deletions(-) diff --git a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto index 9b03169f7dd0..a0da1277bfe2 100644 --- a/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto +++ b/api/envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.proto @@ -49,6 +49,13 @@ message GradientControllerConfig { // The number of requests to aggregate/sample during the minRTT recalculation window before // updating. Defaults to 50. google.protobuf.UInt32Value request_count = 2 [(validate.rules).uint32.gt = 0]; + + // Randomized time delta that will be introduced to the start of the minRTT calculation window. + // This is represented as a percentage of the interval duration. Defaults to 15%. + // + // Example: If the interval is 10s and the jitter is 15%, the next window will begin + // somewhere in the range (10s - 11.5s). + envoy.type.Percent jitter = 3; }; MinimumRTTCalculationParams min_rtt_calc_params = 3 [(validate.rules).message.required = true]; } diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc index 3391c55fb6c3..86594e636083 100644 --- a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc @@ -27,6 +27,9 @@ GradientControllerConfig::GradientControllerConfig( proto_config) : min_rtt_calc_interval_(std::chrono::milliseconds( DurationUtil::durationToMilliseconds(proto_config.min_rtt_calc_params().interval()))), + jitter_pct_( + PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(proto_config.min_rtt_calc_params(), jitter, 15) / + 100.0), sample_rtt_calc_interval_(std::chrono::milliseconds(DurationUtil::durationToMilliseconds( proto_config.concurrency_limit_params().concurrency_update_interval()))), max_concurrency_limit_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( @@ -41,10 +44,12 @@ GradientControllerConfig::GradientControllerConfig( GradientController::GradientController(GradientControllerConfigSharedPtr config, Event::Dispatcher& dispatcher, Runtime::Loader&, - const std::string& stats_prefix, Stats::Scope& scope) + const std::string& stats_prefix, Stats::Scope& scope, + Runtime::RandomGenerator& random) : config_(std::move(config)), dispatcher_(dispatcher), scope_(scope), - stats_(generateStats(scope_, stats_prefix)), deferred_limit_value_(1), num_rq_outstanding_(0), - concurrency_limit_(1), latency_sample_hist_(hist_fast_alloc(), hist_free) { + stats_(generateStats(scope_, stats_prefix)), random_(random), deferred_limit_value_(1), + num_rq_outstanding_(0), concurrency_limit_(1), + latency_sample_hist_(hist_fast_alloc(), hist_free) { min_rtt_calc_timer_ = dispatcher_.createTimer([this]() -> void { enterMinRTTSamplingWindow(); }); sample_reset_timer_ = dispatcher_.createTimer([this]() -> void { @@ -98,7 +103,18 @@ void GradientController::updateMinRTT() { deferred_limit_value_.store(0); } - min_rtt_calc_timer_->enableTimer(config_->minRTTCalcInterval()); + min_rtt_calc_timer_->enableTimer( + applyJitter(config_->minRTTCalcInterval(), config_->jitterPercent())); +} + +std::chrono::milliseconds GradientController::applyJitter(std::chrono::milliseconds interval, + double jitter_pct) const { + if (jitter_pct == 0) { + return interval; + } + + const uint32_t jitter_range_ms = interval.count() * jitter_pct; + return std::chrono::milliseconds(interval.count() + (random_.random() % jitter_range_ms)); } void GradientController::resetSampleWindow() { diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h index a7e27f311467..91566d887a39 100644 --- a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h @@ -44,6 +44,7 @@ class GradientControllerConfig { proto_config); std::chrono::milliseconds minRTTCalcInterval() const { return min_rtt_calc_interval_; } + double jitterPercent() const { return jitter_pct_; } std::chrono::milliseconds sampleRTTCalcInterval() const { return sample_rtt_calc_interval_; } uint32_t maxConcurrencyLimit() const { return max_concurrency_limit_; } uint32_t minRTTAggregateRequestCount() const { return min_rtt_aggregate_request_count_; } @@ -54,6 +55,9 @@ class GradientControllerConfig { // The measured request round-trip time under ideal conditions. const std::chrono::milliseconds min_rtt_calc_interval_; + // Randomized time delta added to the start of the minRTT calculation window. + const double jitter_pct_; + // The measured sample round-trip time from the previous time window. const std::chrono::milliseconds sample_rtt_calc_interval_; @@ -136,8 +140,8 @@ using GradientControllerConfigSharedPtr = std::shared_ptr(makeConfig(yaml_config), *dispatcher_, runtime_, - "test_prefix.", stats_); + "test_prefix.", stats_, random_); } protected: @@ -68,7 +68,7 @@ class GradientControllerTest : public testing::Test { const std::string& yaml_config, std::chrono::milliseconds latency = std::chrono::milliseconds(5)) { const auto config = makeConfig(yaml_config); - for (uint32_t ii = 0; ii <= config->minRTTAggregateRequestCount(); ++ii) { + for (uint32_t i = 0; i <= config->minRTTAggregateRequestCount(); ++i) { tryForward(controller, true); controller->recordLatencySample(latency); } @@ -79,6 +79,7 @@ class GradientControllerTest : public testing::Test { NiceMock runtime_; Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; + NiceMock random_; }; TEST_F(GradientControllerConfigTest, BasicTest) { @@ -88,11 +89,11 @@ TEST_F(GradientControllerConfigTest, BasicTest) { concurrency_limit_params: max_gradient: 2.1 max_concurrency_limit: 1337 - concurrency_update_interval: - nanos: 123000000 + concurrency_update_interval: 0.123s min_rtt_calc_params: - interval: - seconds: 31 + jitter: + value: 13.2 + interval: 31s request_count: 52 )EOF"; @@ -107,17 +108,16 @@ TEST_F(GradientControllerConfigTest, BasicTest) { EXPECT_EQ(config.maxConcurrencyLimit(), 1337); EXPECT_EQ(config.minRTTAggregateRequestCount(), 52); EXPECT_EQ(config.maxGradient(), 2.1); + EXPECT_EQ(config.jitterPercent(), .132); EXPECT_EQ(config.sampleAggregatePercentile(), 0.42); } TEST_F(GradientControllerConfigTest, DefaultValuesTest) { const std::string yaml = R"EOF( concurrency_limit_params: - concurrency_update_interval: - nanos: 123000000 + concurrency_update_interval: 0.123s min_rtt_calc_params: - interval: - seconds: 31 + interval: 31s )EOF"; envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig proto = @@ -131,6 +131,7 @@ TEST_F(GradientControllerConfigTest, DefaultValuesTest) { EXPECT_EQ(config.maxConcurrencyLimit(), 1000); EXPECT_EQ(config.minRTTAggregateRequestCount(), 50); EXPECT_EQ(config.maxGradient(), 2.0); + EXPECT_EQ(config.jitterPercent(), .15); EXPECT_EQ(config.sampleAggregatePercentile(), 0.5); } @@ -141,11 +142,11 @@ TEST_F(GradientControllerTest, MinRTTLogicTest) { concurrency_limit_params: max_gradient: 2.0 max_concurrency_limit: - concurrency_update_interval: - nanos: 100000000 # 100ms + concurrency_update_interval: 0.1s min_rtt_calc_params: - interval: - seconds: 30 + jitter: + value: 0.0 + interval: 30s request_count: 50 )EOF"; @@ -160,7 +161,7 @@ TEST_F(GradientControllerTest, MinRTTLogicTest) { controller->recordLatencySample(min_rtt); // 49 more requests should cause the minRTT to be done calculating. - for (int ii = 0; ii < 49; ++ii) { + for (int i = 0; i < 49; ++i) { EXPECT_EQ(controller->concurrencyLimit(), 1); tryForward(controller, true); tryForward(controller, false); @@ -179,19 +180,19 @@ TEST_F(GradientControllerTest, CancelLatencySample) { concurrency_limit_params: max_gradient: 2.0 max_concurrency_limit: - concurrency_update_interval: - nanos: 100000000 # 100ms + concurrency_update_interval: 0.1s min_rtt_calc_params: - interval: - seconds: 30 + jitter: + value: 0.0 + interval: 30s request_count: 5 )EOF"; auto controller = makeController(yaml); - for (int ii = 1; ii <= 5; ++ii) { + for (int i = 1; i <= 5; ++i) { tryForward(controller, true); - controller->recordLatencySample(std::chrono::milliseconds(ii)); + controller->recordLatencySample(std::chrono::milliseconds(i)); } EXPECT_EQ( 3, stats_.gauge("test_prefix.min_rtt_msecs", Stats::Gauge::ImportMode::NeverImport).value()); @@ -204,11 +205,11 @@ TEST_F(GradientControllerTest, SamplePercentileProcessTest) { concurrency_limit_params: max_gradient: 2.0 max_concurrency_limit: - concurrency_update_interval: - nanos: 100000000 # 100ms + concurrency_update_interval: 0.1s min_rtt_calc_params: - interval: - seconds: 30 + jitter: + value: 0.0 + interval: 30s request_count: 5 )EOF"; @@ -228,11 +229,11 @@ TEST_F(GradientControllerTest, ConcurrencyLimitBehaviorTestBasic) { concurrency_limit_params: max_gradient: 2.0 max_concurrency_limit: - concurrency_update_interval: - nanos: 100000000 # 100ms + concurrency_update_interval: 0.1s min_rtt_calc_params: - interval: - seconds: 30 + jitter: + value: 0.0 + interval: 30s request_count: 5 )EOF"; @@ -253,7 +254,7 @@ TEST_F(GradientControllerTest, ConcurrencyLimitBehaviorTestBasic) { // Ensure that it grows. for (int recalcs = 0; recalcs < 10; ++recalcs) { const auto last_concurrency = controller->concurrencyLimit(); - for (int ii = 1; ii <= 5; ++ii) { + for (int i = 1; i <= 5; ++i) { tryForward(controller, true); controller->recordLatencySample(std::chrono::milliseconds(4)); } @@ -265,7 +266,7 @@ TEST_F(GradientControllerTest, ConcurrencyLimitBehaviorTestBasic) { // Verify that the concurrency limit can now shrink as necessary. for (int recalcs = 0; recalcs < 10; ++recalcs) { const auto last_concurrency = controller->concurrencyLimit(); - for (int ii = 1; ii <= 5; ++ii) { + for (int i = 1; i <= 5; ++i) { tryForward(controller, true); controller->recordLatencySample(std::chrono::milliseconds(6)); } @@ -282,11 +283,11 @@ TEST_F(GradientControllerTest, MaxGradientTest) { concurrency_limit_params: max_gradient: 3.0 max_concurrency_limit: - concurrency_update_interval: - nanos: 100000000 # 100ms + concurrency_update_interval: 0.1s min_rtt_calc_params: - interval: - seconds: 30 + jitter: + value: 0.0 + interval: 30s request_count: 5 )EOF"; @@ -302,7 +303,7 @@ TEST_F(GradientControllerTest, MaxGradientTest) { AllOf(Ge(4950), Le(5050))); // Now verify max gradient value by forcing dramatically faster latency measurements.. - for (int ii = 1; ii <= 5; ++ii) { + for (int i = 1; i <= 5; ++i) { tryForward(controller, true); controller->recordLatencySample(std::chrono::milliseconds(4)); } @@ -319,11 +320,11 @@ TEST_F(GradientControllerTest, MinRTTReturnToPreviousLimit) { concurrency_limit_params: max_gradient: 3.0 max_concurrency_limit: - concurrency_update_interval: - nanos: 100000000 # 100ms + concurrency_update_interval: 0.1s min_rtt_calc_params: - interval: - seconds: 30 + jitter: + value: 0.0 + interval: 30s request_count: 5 )EOF"; @@ -336,7 +337,7 @@ TEST_F(GradientControllerTest, MinRTTReturnToPreviousLimit) { // Force the limit calculation to run a few times from some measurements. for (int sample_iters = 0; sample_iters < 5; ++sample_iters) { const auto last_concurrency = controller->concurrencyLimit(); - for (int ii = 1; ii <= 5; ++ii) { + for (int i = 1; i <= 5; ++i) { tryForward(controller, true); controller->recordLatencySample(std::chrono::milliseconds(4)); } @@ -354,7 +355,7 @@ TEST_F(GradientControllerTest, MinRTTReturnToPreviousLimit) { EXPECT_EQ(controller->concurrencyLimit(), 1); // 49 more requests should cause the minRTT to be done calculating. - for (int ii = 0; ii < 5; ++ii) { + for (int i = 0; i < 5; ++i) { EXPECT_EQ(controller->concurrencyLimit(), 1); tryForward(controller, true); controller->recordLatencySample(std::chrono::milliseconds(13)); @@ -371,11 +372,11 @@ TEST_F(GradientControllerTest, MinRTTRescheduleTest) { concurrency_limit_params: max_gradient: 3.0 max_concurrency_limit: - concurrency_update_interval: - nanos: 100000000 # 100ms + concurrency_update_interval: 0.1s min_rtt_calc_params: - interval: - seconds: 30 + jitter: + value: 0.0 + interval: 30s request_count: 5 )EOF"; @@ -388,7 +389,7 @@ TEST_F(GradientControllerTest, MinRTTRescheduleTest) { // Force the limit calculation to run a few times from some measurements. for (int sample_iters = 0; sample_iters < 5; ++sample_iters) { const auto last_concurrency = controller->concurrencyLimit(); - for (int ii = 1; ii <= 5; ++ii) { + for (int i = 1; i <= 5; ++i) { tryForward(controller, true); controller->recordLatencySample(std::chrono::milliseconds(4)); } @@ -416,11 +417,11 @@ TEST_F(GradientControllerTest, NoSamplesTest) { concurrency_limit_params: max_gradient: 3.0 max_concurrency_limit: - concurrency_update_interval: - nanos: 100000000 # 100ms + concurrency_update_interval: 0.1s min_rtt_calc_params: - interval: - seconds: 30 + jitter: + value: 0.0 + interval: 30s request_count: 5 )EOF"; @@ -433,7 +434,7 @@ TEST_F(GradientControllerTest, NoSamplesTest) { // Force the limit calculation to run a few times from some measurements. for (int sample_iters = 0; sample_iters < 5; ++sample_iters) { const auto last_concurrency = controller->concurrencyLimit(); - for (int ii = 1; ii <= 5; ++ii) { + for (int i = 1; i <= 5; ++i) { tryForward(controller, true); controller->recordLatencySample(std::chrono::milliseconds(4)); } @@ -459,11 +460,11 @@ TEST_F(GradientControllerTest, TimerAccuracyTest) { concurrency_limit_params: max_gradient: 3.0 max_concurrency_limit: - concurrency_update_interval: - nanos: 123000000 # 123ms + concurrency_update_interval: 0.123s min_rtt_calc_params: - interval: - seconds: 45 + jitter: + value: 10.0 + interval: 100s request_count: 5 )EOF"; @@ -479,11 +480,52 @@ TEST_F(GradientControllerTest, TimerAccuracyTest) { .WillOnce(Return(sample_timer)); EXPECT_CALL(*sample_timer, enableTimer(std::chrono::milliseconds(123), _)); auto controller = std::make_shared(makeConfig(yaml), fake_dispatcher, - runtime_, "test_prefix.", stats_); + runtime_, "test_prefix.", stats_, random_); + + // Set the minRTT- this will trigger the timer for the next minRTT calculation. + + // Let's make sure the jitter value can't exceed the configured percentage as well by returning a + // random value > 10% of the interval. + EXPECT_CALL(random_, random()).WillOnce(Return(15000)); + EXPECT_CALL(*rtt_timer, enableTimer(std::chrono::milliseconds(105000), _)); + for (int i = 0; i < 6; ++i) { + tryForward(controller, true); + controller->recordLatencySample(std::chrono::milliseconds(5)); + } +} + +TEST_F(GradientControllerTest, TimerAccuracyTestNoJitter) { + const std::string yaml = R"EOF( +sample_aggregate_percentile: + value: 50 +concurrency_limit_params: + max_gradient: 3.0 + max_concurrency_limit: + concurrency_update_interval: 0.123s +min_rtt_calc_params: + jitter: + value: 0.0 + interval: 45s + request_count: 5 +)EOF"; + + // Verify the configuration affects the timers that are kicked off. + NiceMock fake_dispatcher; + auto sample_timer = new NiceMock; + auto rtt_timer = new NiceMock; + + // Expect the sample timer to trigger start immediately upon controller creation. + EXPECT_CALL(fake_dispatcher, createTimer_(_)) + .Times(2) + .WillOnce(Return(rtt_timer)) + .WillOnce(Return(sample_timer)); + EXPECT_CALL(*sample_timer, enableTimer(std::chrono::milliseconds(123), _)); + auto controller = std::make_shared(makeConfig(yaml), fake_dispatcher, + runtime_, "test_prefix.", stats_, random_); // Set the minRTT- this will trigger the timer for the next minRTT calculation. EXPECT_CALL(*rtt_timer, enableTimer(std::chrono::milliseconds(45000), _)); - for (int ii = 1; ii <= 6; ++ii) { + for (int i = 0; i < 6; ++i) { tryForward(controller, true); controller->recordLatencySample(std::chrono::milliseconds(5)); } From 8387d60a8bc6b030f43a1a7de60588a27b052998 Mon Sep 17 00:00:00 2001 From: Rama Chavali Date: Fri, 27 Sep 2019 21:34:30 +0530 Subject: [PATCH 17/32] config: add proxy version to bootstrap (#8386) Signed-off-by: Rama Chavali --- api/envoy/config/bootstrap/v2/bootstrap.proto | 6 ++++++ api/envoy/config/bootstrap/v3alpha/bootstrap.proto | 6 ++++++ .../configuration/observability/statistics.rst | 2 +- docs/root/intro/version_history.rst | 1 + source/server/server.cc | 12 +++++++++--- test/server/BUILD | 1 + test/server/proxy_version_bootstrap.yaml | 14 ++++++++++++++ test/server/server_test.cc | 10 ++++++++++ 8 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 test/server/proxy_version_bootstrap.yaml diff --git a/api/envoy/config/bootstrap/v2/bootstrap.proto b/api/envoy/config/bootstrap/v2/bootstrap.proto index 07d7285f0060..176138f10c96 100644 --- a/api/envoy/config/bootstrap/v2/bootstrap.proto +++ b/api/envoy/config/bootstrap/v2/bootstrap.proto @@ -18,6 +18,7 @@ import "envoy/config/trace/v2/trace.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; @@ -148,6 +149,11 @@ message Bootstrap { // changes to this string, especially in multi-layer Envoy deployments or deployments using // extensions which are not upstream. string header_prefix = 18; + + // Optional proxy version which will be used to set the value of :ref:`server.version statistic + // ` if specified. Envoy will not process this value, it will be sent as is to + // :ref:. + google.protobuf.UInt64Value stats_server_version_override = 19; } // Administration interface :ref:`operations documentation diff --git a/api/envoy/config/bootstrap/v3alpha/bootstrap.proto b/api/envoy/config/bootstrap/v3alpha/bootstrap.proto index 5a67ee9bef49..0a10d32a4bc0 100644 --- a/api/envoy/config/bootstrap/v3alpha/bootstrap.proto +++ b/api/envoy/config/bootstrap/v3alpha/bootstrap.proto @@ -18,6 +18,7 @@ import "envoy/config/trace/v3alpha/trace.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/struct.proto"; +import "google/protobuf/wrappers.proto"; import "validate/validate.proto"; @@ -144,6 +145,11 @@ message Bootstrap { // changes to this string, especially in multi-layer Envoy deployments or deployments using // extensions which are not upstream. string header_prefix = 18; + + // Optional proxy version which will be used to set the value of :ref:`server.version statistic + // ` if specified. Envoy will not process this value, it will be sent as is to + // :ref:. + google.protobuf.UInt64Value stats_server_version_override = 19; } // Administration interface :ref:`operations documentation diff --git a/docs/root/configuration/observability/statistics.rst b/docs/root/configuration/observability/statistics.rst index 376263f42bb4..89b3bf89761b 100644 --- a/docs/root/configuration/observability/statistics.rst +++ b/docs/root/configuration/observability/statistics.rst @@ -22,7 +22,7 @@ Server related statistics are rooted at *server.* with following statistics: state, Gauge, Current :ref:`State ` of the Server. parent_connections, Gauge, Total connections of the old Envoy process on hot restart total_connections, Gauge, Total connections of both new and old Envoy processes - version, Gauge, Integer represented version number based on SCM revision + version, Gauge, Integer represented version number based on SCM revision or :ref:stats_server_version_override` ` if set. days_until_first_cert_expiring, Gauge, Number of days until the next certificate being managed will expire hot_restart_epoch, Gauge, Current hot restart epoch initialization_time_ms, Histogram, Total time taken for Envoy initialization in milliseconds. This is the time from server start-up until the worker threads are ready to accept new connections diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index be5176d560d2..f43b08d85d9c 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -110,6 +110,7 @@ Version history for each host, useful for DNS based clusters. * api: track and report requests issued since last load report. * build: releases are built with Clang and linked with LLD. +* config: added :ref:stats_server_version_override` ` in bootstrap, that can be used to override :ref:`server.version statistic `. * control-plane: management servers can respond with HTTP 304 to indicate that config is up to date for Envoy proxies polling a :ref:`REST API Config Type ` * csrf: added support for whitelisting additional source origins. * dns: added support for getting DNS record TTL which is used by STRICT_DNS/LOGICAL_DNS cluster as DNS refresh rate. diff --git a/source/server/server.cc b/source/server/server.cc index c6282265df7a..9653cdc8e792 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -309,12 +309,18 @@ void InstanceImpl::initialize(const Options& options, InstanceImpl::failHealthcheck(false); + // Check if bootstrap has server version override set, if yes, we should use that as + // 'server.version' stat. uint64_t version_int; - if (!StringUtil::atoull(VersionInfo::revision().substr(0, 6).c_str(), version_int, 16)) { - throw EnvoyException("compiled GIT SHA is invalid. Invalid build."); + if (bootstrap_.stats_server_version_override().value() > 0) { + version_int = bootstrap_.stats_server_version_override().value(); + } else { + if (!StringUtil::atoull(VersionInfo::revision().substr(0, 6).c_str(), version_int, 16)) { + throw EnvoyException("compiled GIT SHA is invalid. Invalid build."); + } } - server_stats_->version_.set(version_int); + bootstrap_.mutable_node()->set_build_version(VersionInfo::version()); local_info_ = std::make_unique( diff --git a/test/server/BUILD b/test/server/BUILD index 8a18e34f4f57..bae14ad2dd47 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -294,6 +294,7 @@ envoy_cc_test( ":node_bootstrap_no_admin_port.yaml", ":node_bootstrap_with_admin_socket_options.yaml", ":node_bootstrap_without_access_log.yaml", + ":proxy_version_bootstrap.yaml", ":runtime_bootstrap.yaml", ":runtime_test_data", ":static_validation_test_data", diff --git a/test/server/proxy_version_bootstrap.yaml b/test/server/proxy_version_bootstrap.yaml new file mode 100644 index 000000000000..68dcbd2445c1 --- /dev/null +++ b/test/server/proxy_version_bootstrap.yaml @@ -0,0 +1,14 @@ +node: + id: bootstrap_id + cluster: bootstrap_cluster + locality: + zone: bootstrap_zone + sub_zone: bootstrap_sub_zone + build_version: should_be_ignored +admin: + access_log_path: /dev/null + address: + socket_address: + address: {{ ntop_ip_loopback_address }} + port_value: 0 +stats_server_version_override: 100012001 \ No newline at end of file diff --git a/test/server/server_test.cc b/test/server/server_test.cc index cc7a0f69f6c1..b5688754eaa8 100644 --- a/test/server/server_test.cc +++ b/test/server/server_test.cc @@ -316,6 +316,16 @@ TEST_P(ServerInstanceImplTest, StatsFlushWhenServerIsStillInitializing) { server_thread->join(); } +// Validates that the "server.version" is updated with stats_server_version_override from bootstrap. +TEST_P(ServerInstanceImplTest, ProxyVersionOveridesFromBootstrap) { + auto server_thread = startTestServer("test/server/proxy_version_bootstrap.yaml", true); + + EXPECT_EQ(100012001, TestUtility::findGauge(stats_store_, "server.version")->value()); + + server_->dispatcher().post([&] { server_->shutdown(); }); + server_thread->join(); +} + TEST_P(ServerInstanceImplTest, EmptyShutdownLifecycleNotifications) { auto server_thread = startTestServer("test/server/node_bootstrap.yaml", false); server_->dispatcher().post([&] { server_->shutdown(); }); From 72e5a2b25498ecac84f976277319f4bfe9d549d7 Mon Sep 17 00:00:00 2001 From: Derek Argueta Date: Fri, 27 Sep 2019 09:06:00 -0700 Subject: [PATCH 18/32] clang-tidy: check casing on type aliases (#8408) Signed-off-by: Derek Argueta --- .clang-tidy | 3 +++ source/common/common/hash.h | 4 ++-- source/common/common/logger.h | 2 +- source/common/common/utility.h | 4 ++-- source/common/stats/allocator_impl.h | 4 ++-- source/common/stats/symbol_table_impl.h | 10 +++++----- source/extensions/common/wasm/wasm_vm.h | 11 ++++++++--- source/extensions/filters/network/kafka/parser.h | 6 +++--- test/common/http/http2/http2_frame.cc | 4 +++- test/common/http/http2/http2_frame.h | 12 ++++++------ test/tools/router_check/router.cc | 8 ++++---- 11 files changed, 39 insertions(+), 29 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index a53d3ff5ea23..02b0a031e98e 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -64,3 +64,6 @@ CheckOptions: - key: readability-identifier-naming.ParameterCase value: 'lower_case' + + - key: readability-identifier-naming.TypeAliasCase + value: 'CamelCase' diff --git a/source/common/common/hash.h b/source/common/common/hash.h index c9fcd00909cb..8be939356001 100644 --- a/source/common/common/hash.h +++ b/source/common/common/hash.h @@ -101,7 +101,7 @@ struct HeterogeneousStringHash { // https://en.cppreference.com/w/cpp/utility/functional/less_void for an // official reference, and https://abseil.io/tips/144 for a description of // using it in the context of absl. - using is_transparent = void; + using is_transparent = void; // NOLINT(readability-identifier-naming) size_t operator()(absl::string_view a) const { return HashUtil::xxHash64(a); } size_t operator()(const SharedString& a) const { return HashUtil::xxHash64(*a); } @@ -109,7 +109,7 @@ struct HeterogeneousStringHash { struct HeterogeneousStringEqual { // See description for HeterogeneousStringHash::is_transparent. - using is_transparent = void; + using is_transparent = void; // NOLINT(readability-identifier-naming) size_t operator()(absl::string_view a, absl::string_view b) const { return a == b; } size_t operator()(const SharedString& a, const SharedString& b) const { return *a == *b; } diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 58f06b447f0c..5efa9accd340 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -79,7 +79,7 @@ class Logger { * but the method to log at err level is called LOGGER.error not LOGGER.err. All other level are * fine spdlog::info corresponds to LOGGER.info method. */ - using levels = enum { + using Levels = enum { trace = spdlog::level::trace, debug = spdlog::level::debug, info = spdlog::level::info, diff --git a/source/common/common/utility.h b/source/common/common/utility.h index d5faf18a596d..150aaf79624c 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -152,7 +152,7 @@ class StringUtil { */ struct CaseInsensitiveCompare { // Enable heterogeneous lookup (https://abseil.io/tips/144) - using is_transparent = void; + using is_transparent = void; // NOLINT(readability-identifier-naming) bool operator()(absl::string_view lhs, absl::string_view rhs) const; }; @@ -163,7 +163,7 @@ class StringUtil { */ struct CaseInsensitiveHash { // Enable heterogeneous lookup (https://abseil.io/tips/144) - using is_transparent = void; + using is_transparent = void; // NOLINT(readability-identifier-naming) uint64_t operator()(absl::string_view key) const; }; diff --git a/source/common/stats/allocator_impl.h b/source/common/stats/allocator_impl.h index 5d5e82721b6d..02a799459910 100644 --- a/source/common/stats/allocator_impl.h +++ b/source/common/stats/allocator_impl.h @@ -36,13 +36,13 @@ class AllocatorImpl : public Allocator { private: struct HeapStatHash { - using is_transparent = void; + using is_transparent = void; // NOLINT(readability-identifier-naming) size_t operator()(const Metric* a) const { return a->statName().hash(); } size_t operator()(StatName a) const { return a.hash(); } }; struct HeapStatCompare { - using is_transparent = void; + using is_transparent = void; // NOLINT(readability-identifier-naming) bool operator()(const Metric* a, const Metric* b) const { return a->statName() == b->statName(); } diff --git a/source/common/stats/symbol_table_impl.h b/source/common/stats/symbol_table_impl.h index d047708fe8d9..d0317b9870ff 100644 --- a/source/common/stats/symbol_table_impl.h +++ b/source/common/stats/symbol_table_impl.h @@ -569,7 +569,7 @@ struct HeterogeneousStatNameHash { // https://en.cppreference.com/w/cpp/utility/functional/less_void for an // official reference, and https://abseil.io/tips/144 for a description of // using it in the context of absl. - using is_transparent = void; + using is_transparent = void; // NOLINT(readability-identifier-naming) size_t operator()(StatName a) const { return a.hash(); } size_t operator()(const StatNameStorage& a) const { return a.statName().hash(); } @@ -577,7 +577,7 @@ struct HeterogeneousStatNameHash { struct HeterogeneousStatNameEqual { // See description for HeterogeneousStatNameHash::is_transparent. - using is_transparent = void; + using is_transparent = void; // NOLINT(readability-identifier-naming) size_t operator()(StatName a, StatName b) const { return a == b; } size_t operator()(const StatNameStorage& a, const StatNameStorage& b) const { @@ -598,7 +598,7 @@ class StatNameStorageSet { public: using HashSet = absl::flat_hash_set; - using iterator = HashSet::iterator; + using Iterator = HashSet::iterator; ~StatNameStorageSet(); @@ -620,12 +620,12 @@ class StatNameStorageSet { * @param stat_name The stat_name to find. * @return the iterator pointing to the stat_name, or end() if not found. */ - iterator find(StatName stat_name) { return hash_set_.find(stat_name); } + Iterator find(StatName stat_name) { return hash_set_.find(stat_name); } /** * @return the end-marker. */ - iterator end() { return hash_set_.end(); } + Iterator end() { return hash_set_.end(); } /** * @param set the storage set to swap with. diff --git a/source/extensions/common/wasm/wasm_vm.h b/source/extensions/common/wasm/wasm_vm.h index 42e5a08b0d33..e01312ca807f 100644 --- a/source/extensions/common/wasm/wasm_vm.h +++ b/source/extensions/common/wasm/wasm_vm.h @@ -24,8 +24,12 @@ struct Word { }; // Convert Word type for use by 32-bit VMs. -template struct ConvertWordTypeToUint32 { using type = T; }; -template <> struct ConvertWordTypeToUint32 { using type = uint32_t; }; +template struct ConvertWordTypeToUint32 { + using type = T; // NOLINT(readability-identifier-naming) +}; +template <> struct ConvertWordTypeToUint32 { + using type = uint32_t; // NOLINT(readability-identifier-naming) +}; // Convert Word-based function types for 32-bit VMs. template struct ConvertFunctionTypeWordToUint32 {}; @@ -55,13 +59,14 @@ struct WasmFuncTypeHelper {}; template struct WasmFuncTypeHelper { + // NOLINTNEXTLINE(readability-identifier-naming) using type = typename WasmFuncTypeHelper::type; }; template struct WasmFuncTypeHelper<0, ReturnType, ContextType, ParamType, ReturnType(ContextType, Args...)> { - using type = ReturnType(ContextType, Args...); + using type = ReturnType(ContextType, Args...); // NOLINT(readability-identifier-naming) }; template diff --git a/source/extensions/filters/network/kafka/parser.h b/source/extensions/filters/network/kafka/parser.h index f05a07bb3a84..f131a3242659 100644 --- a/source/extensions/filters/network/kafka/parser.h +++ b/source/extensions/filters/network/kafka/parser.h @@ -43,7 +43,7 @@ using ParserSharedPtr = std::shared_ptr>; */ template class ParseResponse { public: - using failure_type = FailureDataType; + using FailureType = FailureDataType; /** * Constructs a response that states that parser still needs data and should not be replaced. @@ -99,8 +99,8 @@ template class AbstractSentinelPar data = {data.data() + min, data.size() - min}; context_->remaining() -= min; if (0 == context_->remaining()) { - using failure_type = typename ResponseType::failure_type::element_type; - auto failure_data = std::make_shared(context_->asFailureData()); + using FailureType = typename ResponseType::FailureType::element_type; + auto failure_data = std::make_shared(context_->asFailureData()); return ResponseType::parseFailure(failure_data); } else { return ResponseType::stillWaiting(); diff --git a/test/common/http/http2/http2_frame.cc b/test/common/http/http2/http2_frame.cc index 368630e1f6ec..8667888eff82 100644 --- a/test/common/http/http2/http2_frame.cc +++ b/test/common/http/http2/http2_frame.cc @@ -10,7 +10,9 @@ namespace { uint32_t makeRequestStreamId(uint32_t stream_id) { return htonl((stream_id << 1) | 1); } // All this templatized stuff is for the typesafe constexpr bitwise ORing of the "enum class" values -template struct FirstArgType { using type = First; }; +template struct FirstArgType { + using type = First; // NOLINT(readability-identifier-naming) +}; template constexpr uint8_t orFlags(Flag flag) { return static_cast(flag); } diff --git a/test/common/http/http2/http2_frame.h b/test/common/http/http2/http2_frame.h index 52b838dbb987..7ef923ca23d4 100644 --- a/test/common/http/http2/http2_frame.h +++ b/test/common/http/http2/http2_frame.h @@ -19,8 +19,8 @@ class Http2Frame { public: Http2Frame() = default; - using iterator = DataContainer::iterator; - using const_iterator = DataContainer::const_iterator; + using Iterator = DataContainer::iterator; + using ConstIterator = DataContainer::const_iterator; static constexpr size_t HeaderSize = 9; static const char Preamble[25]; @@ -110,10 +110,10 @@ class Http2Frame { size_t size() const { return data_.size(); } // Access to the raw frame bytes const uint8_t* data() const { return data_.data(); } - iterator begin() { return data_.begin(); } - iterator end() { return data_.end(); } - const_iterator begin() const { return data_.begin(); } - const_iterator end() const { return data_.end(); } + Iterator begin() { return data_.begin(); } + Iterator end() { return data_.end(); } + ConstIterator begin() const { return data_.begin(); } + ConstIterator end() const { return data_.end(); } bool empty() const { return data_.empty(); } private: diff --git a/test/tools/router_check/router.cc b/test/tools/router_check/router.cc index b358dab9182c..6c5ea647a6c1 100644 --- a/test/tools/router_check/router.cc +++ b/test/tools/router_check/router.cc @@ -119,8 +119,8 @@ bool RouterCheckTool::compareEntriesInJson(const std::string& expected_route_jso std::string test_name = check_config->getString("test_name", ""); tests_.emplace_back(test_name, std::vector{}); Json::ObjectSharedPtr validate = check_config->getObject("validate"); - using checkerFunc = std::function; - const std::unordered_map checkers = { + using CheckerFunc = std::function; + const std::unordered_map checkers = { {"cluster_name", [this](auto&... params) -> bool { return this->compareCluster(params...); }}, {"virtual_cluster_name", @@ -189,9 +189,9 @@ bool RouterCheckTool::compareEntries(const std::string& expected_routes) { tests_.emplace_back(test_name, std::vector{}); const envoy::RouterCheckToolSchema::ValidationAssert& validate = check_config.validate(); - using checkerFunc = + using CheckerFunc = std::function; - checkerFunc checkers[] = { + CheckerFunc checkers[] = { [this](auto&... params) -> bool { return this->compareCluster(params...); }, [this](auto&... params) -> bool { return this->compareVirtualCluster(params...); }, [this](auto&... params) -> bool { return this->compareVirtualHost(params...); }, From 4606bc1e813a51678b14c7830008f7e86640e378 Mon Sep 17 00:00:00 2001 From: "Tejasvi (Teju) Nareddy" Date: Fri, 27 Sep 2019 09:07:10 -0700 Subject: [PATCH 19/32] router: add more upstream request metadata to tracing spans (#8289) Signed-off-by: Teju Nareddy --- source/common/http/conn_manager_impl.cc | 6 +- source/common/router/router.cc | 22 ++- source/common/tracing/http_tracer_impl.cc | 41 ++++-- source/common/tracing/http_tracer_impl.h | 30 +++- test/common/router/router_test.cc | 136 ++++++++++++++++++- test/common/tracing/http_tracer_impl_test.cc | 38 +++--- 6 files changed, 231 insertions(+), 42 deletions(-) diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index cdd68ebcc572..7285309ed21a 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -520,9 +520,9 @@ ConnectionManagerImpl::ActiveStream::~ActiveStream() { } if (active_span_) { - Tracing::HttpTracerUtility::finalizeSpan(*active_span_, request_headers_.get(), - response_headers_.get(), response_trailers_.get(), - stream_info_, *this); + Tracing::HttpTracerUtility::finalizeDownstreamSpan( + *active_span_, request_headers_.get(), response_headers_.get(), response_trailers_.get(), + stream_info_, *this); } if (state_.successful_upgrade_) { connection_manager_.stats_.named_.downstream_cx_upgrades_active_.dec(); diff --git a/source/common/router/router.cc b/source/common/router/router.cc index 412bed0392b7..05a7ca48112e 100644 --- a/source/common/router/router.cc +++ b/source/common/router/router.cc @@ -1332,7 +1332,10 @@ Filter::UpstreamRequest::UpstreamRequest(Filter& parent, Http::ConnectionPool::I span_ = parent_.callbacks_->activeSpan().spawnChild( parent_.callbacks_->tracingConfig(), "router " + parent.cluster_->name() + " egress", parent.timeSource().systemTime()); - span_->setTag(Tracing::Tags::get().Component, Tracing::Tags::get().Proxy); + if (parent.attempt_count_ != 1) { + // This is a retry request, add this metadata to span. + span_->setTag(Tracing::Tags::get().RetryCount, std::to_string(parent.attempt_count_ - 1)); + } } stream_info_.healthCheck(parent_.callbacks_->streamInfo().healthCheck()); @@ -1340,9 +1343,11 @@ Filter::UpstreamRequest::UpstreamRequest(Filter& parent, Http::ConnectionPool::I Filter::UpstreamRequest::~UpstreamRequest() { if (span_ != nullptr) { - // TODO(mattklein123): Add tags based on what happened to this request (retries, reset, etc.). - span_->finishSpan(); + Tracing::HttpTracerUtility::finalizeUpstreamSpan(*span_, upstream_headers_.get(), + upstream_trailers_.get(), stream_info_, + Tracing::EgressConfig::get()); } + if (per_try_timeout_ != nullptr) { // Allows for testing. per_try_timeout_->disableTimer(); @@ -1481,6 +1486,12 @@ void Filter::UpstreamRequest::onResetStream(Http::StreamResetReason reason, absl::string_view transport_failure_reason) { ScopeTrackerScopeState scope(&parent_.callbacks_->scope(), parent_.callbacks_->dispatcher()); + if (span_ != nullptr) { + // Add tags about reset. + span_->setTag(Tracing::Tags::get().Error, Tracing::Tags::get().True); + span_->setTag(Tracing::Tags::get().ErrorReason, Http::Utility::resetReasonToString(reason)); + } + clearRequestEncoder(); awaiting_headers_ = false; if (!calling_encode_headers_) { @@ -1497,6 +1508,11 @@ void Filter::UpstreamRequest::resetStream() { return; } + if (span_ != nullptr) { + // Add tags about the cancellation. + span_->setTag(Tracing::Tags::get().Canceled, Tracing::Tags::get().True); + } + if (conn_pool_stream_handle_) { ENVOY_STREAM_LOG(debug, "cancelling pool request", *parent_.callbacks_); ASSERT(!request_encoder_); diff --git a/source/common/tracing/http_tracer_impl.cc b/source/common/tracing/http_tracer_impl.cc index 35bee4ff7de1..a1e3ea860e74 100644 --- a/source/common/tracing/http_tracer_impl.cc +++ b/source/common/tracing/http_tracer_impl.cc @@ -138,11 +138,11 @@ static void annotateVerbose(Span& span, const StreamInfo::StreamInfo& stream_inf } } -void HttpTracerUtility::finalizeSpan(Span& span, const Http::HeaderMap* request_headers, - const Http::HeaderMap* response_headers, - const Http::HeaderMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, - const Config& tracing_config) { +void HttpTracerUtility::finalizeDownstreamSpan(Span& span, const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, + const StreamInfo::StreamInfo& stream_info, + const Config& tracing_config) { // Pre response data. if (request_headers) { if (request_headers->RequestId()) { @@ -173,6 +173,31 @@ void HttpTracerUtility::finalizeSpan(Span& span, const Http::HeaderMap* request_ } } span.setTag(Tracing::Tags::get().RequestSize, std::to_string(stream_info.bytesReceived())); + span.setTag(Tracing::Tags::get().ResponseSize, std::to_string(stream_info.bytesSent())); + + setCommonTags(span, response_headers, response_trailers, stream_info, tracing_config); + + span.finishSpan(); +} + +void HttpTracerUtility::finalizeUpstreamSpan(Span& span, const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, + const StreamInfo::StreamInfo& stream_info, + const Config& tracing_config) { + span.setTag(Tracing::Tags::get().HttpProtocol, + AccessLog::AccessLogFormatUtils::protocolToString(stream_info.protocol())); + + setCommonTags(span, response_headers, response_trailers, stream_info, tracing_config); + + span.finishSpan(); +} + +void HttpTracerUtility::setCommonTags(Span& span, const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, + const StreamInfo::StreamInfo& stream_info, + const Config& tracing_config) { + + span.setTag(Tracing::Tags::get().Component, Tracing::Tags::get().Proxy); if (nullptr != stream_info.upstreamHost()) { span.setTag(Tracing::Tags::get().UpstreamCluster, stream_info.upstreamHost()->cluster().name()); @@ -180,7 +205,6 @@ void HttpTracerUtility::finalizeSpan(Span& span, const Http::HeaderMap* request_ // Post response data. span.setTag(Tracing::Tags::get().HttpStatusCode, buildResponseCode(stream_info)); - span.setTag(Tracing::Tags::get().ResponseSize, std::to_string(stream_info.bytesSent())); span.setTag(Tracing::Tags::get().ResponseFlags, StreamInfo::ResponseFlagUtils::toShortString(stream_info)); @@ -198,8 +222,6 @@ void HttpTracerUtility::finalizeSpan(Span& span, const Http::HeaderMap* request_ if (!stream_info.responseCode() || Http::CodeUtility::is5xx(stream_info.responseCode().value())) { span.setTag(Tracing::Tags::get().Error, Tracing::Tags::get().True); } - - span.finishSpan(); } HttpTracerImpl::HttpTracerImpl(DriverPtr&& driver, const LocalInfo::LocalInfo& local_info) @@ -217,8 +239,9 @@ SpanPtr HttpTracerImpl::startSpan(const Config& config, Http::HeaderMap& request SpanPtr active_span = driver_->startSpan(config, request_headers, span_name, stream_info.startTime(), tracing_decision); + + // Set tags related to the local environment if (active_span) { - active_span->setTag(Tracing::Tags::get().Component, Tracing::Tags::get().Proxy); active_span->setTag(Tracing::Tags::get().NodeId, local_info_.nodeName()); active_span->setTag(Tracing::Tags::get().Zone, local_info_.zoneName()); } diff --git a/source/common/tracing/http_tracer_impl.h b/source/common/tracing/http_tracer_impl.h index e7560acac0ea..bb99d7934e77 100644 --- a/source/common/tracing/http_tracer_impl.h +++ b/source/common/tracing/http_tracer_impl.h @@ -40,6 +40,7 @@ class TracingTagValues { // Non-standard tag names. const std::string DownstreamCluster = "downstream_cluster"; + const std::string ErrorReason = "error.reason"; const std::string GrpcStatusCode = "grpc.status_code"; const std::string GrpcMessage = "grpc.message"; const std::string GuidXClientTraceId = "guid:x-client-trace-id"; @@ -49,6 +50,7 @@ class TracingTagValues { const std::string RequestSize = "request_size"; const std::string ResponseFlags = "response_flags"; const std::string ResponseSize = "response_size"; + const std::string RetryCount = "retry.count"; const std::string Status = "status"; const std::string UpstreamCluster = "upstream_cluster"; const std::string UserAgent = "user_agent"; @@ -98,13 +100,29 @@ class HttpTracerUtility { const Http::HeaderMap& request_headers); /** - * 1) Fill in span tags based on the response headers. - * 2) Finish active span. + * Adds information obtained from the downstream request headers as tags to the active span. + * Then finishes the span. */ - static void finalizeSpan(Span& span, const Http::HeaderMap* request_headers, - const Http::HeaderMap* response_headers, - const Http::HeaderMap* response_trailers, - const StreamInfo::StreamInfo& stream_info, const Config& tracing_config); + static void finalizeDownstreamSpan(Span& span, const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, + const StreamInfo::StreamInfo& stream_info, + const Config& tracing_config); + + /** + * Adds information obtained from the upstream request headers as tags to the active span. + * Then finishes the span. + */ + static void finalizeUpstreamSpan(Span& span, const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, + const StreamInfo::StreamInfo& stream_info, + const Config& tracing_config); + +private: + static void setCommonTags(Span& span, const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, + const StreamInfo::StreamInfo& stream_info, + const Config& tracing_config); static const std::string IngressOperation; static const std::string EgressOperation; diff --git a/test/common/router/router_test.cc b/test/common/router/router_test.cc index 544ec1062904..c05e23793075 100644 --- a/test/common/router/router_test.cc +++ b/test/common/router/router_test.cc @@ -4400,6 +4400,7 @@ class RouterTestChildSpan : public RouterTestBase { }; // Make sure child spans start/inject/finish with a normal flow. +// An upstream request succeeds and a single span is created. TEST_F(RouterTestChildSpan, BasicFlow) { EXPECT_CALL(callbacks_.route_->route_entry_, timeout()) .WillOnce(Return(std::chrono::milliseconds(0))); @@ -4422,16 +4423,22 @@ TEST_F(RouterTestChildSpan, BasicFlow) { EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router fake_cluster egress", _)) .WillOnce(Return(child_span)); EXPECT_CALL(callbacks_, tracingConfig()); - EXPECT_CALL(*child_span, - setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); router_.decodeHeaders(headers, true); Http::HeaderMapPtr response_headers(new Http::TestHeaderMapImpl{{":status", "200"}}); + EXPECT_CALL(*child_span, + setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/1.0"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), Eq("fake_cluster"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("200"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().ResponseFlags), Eq("-"))); EXPECT_CALL(*child_span, finishSpan()); response_decoder->decodeHeaders(std::move(response_headers), true); } // Make sure child spans start/inject/finish with a reset flow. +// The upstream responds back to envoy before the reset, so the span has fields that represent a +// response and reset. TEST_F(RouterTestChildSpan, ResetFlow) { EXPECT_CALL(callbacks_.route_->route_entry_, timeout()) .WillOnce(Return(std::chrono::milliseconds(0))); @@ -4454,17 +4461,138 @@ TEST_F(RouterTestChildSpan, ResetFlow) { EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router fake_cluster egress", _)) .WillOnce(Return(child_span)); EXPECT_CALL(callbacks_, tracingConfig()); - EXPECT_CALL(*child_span, - setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); router_.decodeHeaders(headers, true); + // Upstream responds back to envoy. Http::HeaderMapPtr response_headers(new Http::TestHeaderMapImpl{{":status", "200"}}); response_decoder->decodeHeaders(std::move(response_headers), false); + // The reset occurs after the upstream response, so the span has a valid status code but also an + // error. + EXPECT_CALL(*child_span, + setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/1.0"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), Eq("fake_cluster"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("200"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().ResponseFlags), Eq("UR"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().Error), Eq(Tracing::Tags::get().True))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().ErrorReason), Eq("remote reset"))); EXPECT_CALL(*child_span, finishSpan()); encoder.stream_.resetStream(Http::StreamResetReason::RemoteReset); } +// Make sure child spans start/inject/finish with a cancellation flow. +// An upstream request is created but is then cancelled before. The resulting span has the +// cancellation fields. +TEST_F(RouterTestChildSpan, CancelFlow) { + EXPECT_CALL(callbacks_.route_->route_entry_, timeout()) + .WillOnce(Return(std::chrono::milliseconds(0))); + EXPECT_CALL(callbacks_.dispatcher_, createTimer_(_)).Times(0); + + NiceMock encoder; + Tracing::MockSpan* child_span{new Tracing::MockSpan()}; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder&, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + EXPECT_CALL(*child_span, injectContext(_)); + callbacks.onPoolReady(encoder, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + Http::TestHeaderMapImpl headers; + HttpTestUtility::addDefaultHeaders(headers); + EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router fake_cluster egress", _)) + .WillOnce(Return(child_span)); + EXPECT_CALL(callbacks_, tracingConfig()); + router_.decodeHeaders(headers, true); + + // Destroy the router, causing the upstream request to be cancelled. + // Response code on span is 0 because the upstream never sent a response. + EXPECT_CALL(*child_span, + setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/1.0"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), Eq("fake_cluster"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("0"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().ResponseFlags), Eq("-"))); + EXPECT_CALL(*child_span, setTag(Eq(Tracing::Tags::get().Error), Eq(Tracing::Tags::get().True))); + EXPECT_CALL(*child_span, + setTag(Eq(Tracing::Tags::get().Canceled), Eq(Tracing::Tags::get().True))); + EXPECT_CALL(*child_span, finishSpan()); + router_.onDestroy(); +} + +// Make sure child spans start/inject/finish with retry flow. +// The first request will fail because of an upstream reset, so the span will be annotated with the +// reset reason. The second request will succeed, so the span will be annotated with 200 OK. +TEST_F(RouterTestChildSpan, ResetRetryFlow) { + NiceMock encoder1; + Http::StreamDecoder* response_decoder = nullptr; + Tracing::MockSpan* child_span_1{new Tracing::MockSpan()}; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + EXPECT_CALL(*child_span_1, injectContext(_)); + callbacks.onPoolReady(encoder1, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + expectResponseTimerCreate(); + + // Upstream responds back to envoy simulating an upstream reset. + Http::TestHeaderMapImpl headers{{"x-envoy-retry-on", "5xx"}, {"x-envoy-internal", "true"}}; + HttpTestUtility::addDefaultHeaders(headers); + EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router fake_cluster egress", _)) + .WillOnce(Return(child_span_1)); + EXPECT_CALL(callbacks_, tracingConfig()); + router_.decodeHeaders(headers, true); + + // The span should be annotated with the reset-related fields. + EXPECT_CALL(*child_span_1, + setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); + EXPECT_CALL(*child_span_1, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/1.0"))); + EXPECT_CALL(*child_span_1, setTag(Eq(Tracing::Tags::get().UpstreamCluster), Eq("fake_cluster"))); + EXPECT_CALL(*child_span_1, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("0"))); + EXPECT_CALL(*child_span_1, setTag(Eq(Tracing::Tags::get().ResponseFlags), Eq("UR"))); + EXPECT_CALL(*child_span_1, setTag(Eq(Tracing::Tags::get().Error), Eq(Tracing::Tags::get().True))) + .Times(2); + EXPECT_CALL(*child_span_1, setTag(Eq(Tracing::Tags::get().ErrorReason), Eq("remote reset"))); + EXPECT_CALL(*child_span_1, finishSpan()); + + router_.retry_state_->expectResetRetry(); + encoder1.stream_.resetStream(Http::StreamResetReason::RemoteReset); + + // We expect this reset to kick off a new request. + NiceMock encoder2; + Tracing::MockSpan* child_span_2{new Tracing::MockSpan()}; + EXPECT_CALL(cm_.conn_pool_, newStream(_, _)) + .WillOnce(Invoke([&](Http::StreamDecoder& decoder, Http::ConnectionPool::Callbacks& callbacks) + -> Http::ConnectionPool::Cancellable* { + response_decoder = &decoder; + EXPECT_CALL(*child_span_2, injectContext(_)); + EXPECT_CALL(*router_.retry_state_, onHostAttempted(_)); + callbacks.onPoolReady(encoder2, cm_.conn_pool_.host_, upstream_stream_info_); + return nullptr; + })); + + EXPECT_CALL(callbacks_.active_span_, spawnChild_(_, "router fake_cluster egress", _)) + .WillOnce(Return(child_span_2)); + EXPECT_CALL(callbacks_, tracingConfig()); + EXPECT_CALL(*child_span_2, setTag(Eq(Tracing::Tags::get().RetryCount), Eq("1"))); + + router_.retry_state_->callback_(); + + // Upstream responds back with a normal response. Span should be annotated as usual. + Http::HeaderMapPtr response_headers(new Http::TestHeaderMapImpl{{":status", "200"}}); + EXPECT_CALL(*child_span_2, + setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); + EXPECT_CALL(*child_span_2, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/1.0"))); + EXPECT_CALL(*child_span_2, setTag(Eq(Tracing::Tags::get().UpstreamCluster), Eq("fake_cluster"))); + EXPECT_CALL(*child_span_2, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("200"))); + EXPECT_CALL(*child_span_2, setTag(Eq(Tracing::Tags::get().ResponseFlags), Eq("-"))); + EXPECT_CALL(*child_span_2, finishSpan()); + response_decoder->decodeHeaders(std::move(response_headers), true); +} + Protobuf::RepeatedPtrField protobufStrList(const std::vector& v) { Protobuf::RepeatedPtrField res; for (auto& field : v) { diff --git a/test/common/tracing/http_tracer_impl_test.cc b/test/common/tracing/http_tracer_impl_test.cc index 496da50b5873..a35f4cb1cafb 100644 --- a/test/common/tracing/http_tracer_impl_test.cc +++ b/test/common/tracing/http_tracer_impl_test.cc @@ -137,8 +137,8 @@ TEST(HttpConnManFinalizerImpl, OriginalAndLongPath) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/2"))); NiceMock config; - HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, - stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, &request_headers, &response_headers, + &response_trailers, stream_info, config); } TEST(HttpConnManFinalizerImpl, NoGeneratedId) { @@ -166,8 +166,8 @@ TEST(HttpConnManFinalizerImpl, NoGeneratedId) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpProtocol), Eq("HTTP/2"))); NiceMock config; - HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, - stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, &request_headers, &response_headers, + &response_trailers, stream_info, config); } TEST(HttpConnManFinalizerImpl, NullRequestHeaders) { @@ -185,10 +185,11 @@ TEST(HttpConnManFinalizerImpl, NullRequestHeaders) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().ResponseSize), Eq("11"))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().ResponseFlags), Eq("-"))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().RequestSize), Eq("10"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), _)).Times(0); NiceMock config; - HttpTracerUtility::finalizeSpan(span, nullptr, nullptr, nullptr, stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, nullptr, nullptr, nullptr, stream_info, config); } TEST(HttpConnManFinalizerImpl, StreamInfoLogs) { @@ -226,7 +227,7 @@ TEST(HttpConnManFinalizerImpl, StreamInfoLogs) { NiceMock config; EXPECT_CALL(config, verbose).WillOnce(Return(true)); - HttpTracerUtility::finalizeSpan(span, nullptr, nullptr, nullptr, stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, nullptr, nullptr, nullptr, stream_info, config); } TEST(HttpConnManFinalizerImpl, UpstreamClusterTagSet) { @@ -240,6 +241,7 @@ TEST(HttpConnManFinalizerImpl, UpstreamClusterTagSet) { EXPECT_CALL(stream_info, responseCode()).WillRepeatedly(ReturnPointee(&response_code)); EXPECT_CALL(stream_info, upstreamHost()).Times(2); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), Eq("my_upstream_cluster"))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("0"))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().Error), Eq(Tracing::Tags::get().True))); @@ -248,7 +250,7 @@ TEST(HttpConnManFinalizerImpl, UpstreamClusterTagSet) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().RequestSize), Eq("10"))); NiceMock config; - HttpTracerUtility::finalizeSpan(span, nullptr, nullptr, nullptr, stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, nullptr, nullptr, nullptr, stream_info, config); } TEST(HttpConnManFinalizerImpl, SpanOptionalHeaders) { @@ -285,11 +287,12 @@ TEST(HttpConnManFinalizerImpl, SpanOptionalHeaders) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().Error), Eq(Tracing::Tags::get().True))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().ResponseSize), Eq("100"))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().ResponseFlags), Eq("-"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), _)).Times(0); NiceMock config; - HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, - stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, &request_headers, &response_headers, + &response_trailers, stream_info, config); } TEST(HttpConnManFinalizerImpl, SpanPopulatedFailureResponse) { @@ -347,10 +350,11 @@ TEST(HttpConnManFinalizerImpl, SpanPopulatedFailureResponse) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().HttpStatusCode), Eq("503"))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().ResponseSize), Eq("100"))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().ResponseFlags), Eq("UT"))); + EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().Component), Eq(Tracing::Tags::get().Proxy))); EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().UpstreamCluster), _)).Times(0); - HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, - stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, &request_headers, &response_headers, + &response_trailers, stream_info, config); } TEST(HttpConnManFinalizerImpl, GrpcOkStatus) { @@ -384,8 +388,8 @@ TEST(HttpConnManFinalizerImpl, GrpcOkStatus) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().GrpcMessage), Eq(""))); NiceMock config; - HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, - stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, &request_headers, &response_headers, + &response_trailers, stream_info, config); } TEST(HttpConnManFinalizerImpl, GrpcErrorTag) { @@ -421,8 +425,8 @@ TEST(HttpConnManFinalizerImpl, GrpcErrorTag) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().GrpcMessage), Eq("permission denied"))); NiceMock config; - HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, - stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, &request_headers, &response_headers, + &response_trailers, stream_info, config); } TEST(HttpConnManFinalizerImpl, GrpcTrailersOnly) { @@ -459,8 +463,8 @@ TEST(HttpConnManFinalizerImpl, GrpcTrailersOnly) { EXPECT_CALL(span, setTag(Eq(Tracing::Tags::get().GrpcMessage), Eq("permission denied"))); NiceMock config; - HttpTracerUtility::finalizeSpan(span, &request_headers, &response_headers, &response_trailers, - stream_info, config); + HttpTracerUtility::finalizeDownstreamSpan(span, &request_headers, &response_headers, + &response_trailers, stream_info, config); } TEST(HttpTracerUtilityTest, operationTypeToString) { From ef9139a991fb9f7df676430f626b32db93a06932 Mon Sep 17 00:00:00 2001 From: asraa Date: Fri, 27 Sep 2019 14:40:33 -0400 Subject: [PATCH 20/32] protobuf: add back ubsan patch (#8417) Adds back protobuf patch fixing UBSAN error (protocolbuffers/protobuf#6333). This was removed on updating protobuf to 3.9.1, but will be included in the protobuf 3.10 release. Fixes OSS-Fuzz issue: https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=17759 Testing: Added corpus Signed-off-by: Asra Ali --- bazel/protobuf.patch | 20 +++++++++++++++++++ bazel/repositories.bzl | 4 ++-- ...minimized-route_fuzz_test-5650952886943744 | 11 ++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5650952886943744 diff --git a/bazel/protobuf.patch b/bazel/protobuf.patch index 3a26bb356f97..d00e39ef9412 100644 --- a/bazel/protobuf.patch +++ b/bazel/protobuf.patch @@ -1,3 +1,23 @@ +diff --git a/src/google/protobuf/stubs/strutil.cc b/src/google/protobuf/stubs/strutil.cc +index 3844fa6b8b..5486887295 100644 +--- a/src/google/protobuf/stubs/strutil.cc ++++ b/src/google/protobuf/stubs/strutil.cc +@@ -1065,10 +1065,12 @@ char* FastUInt32ToBufferLeft(uint32 u, char* buffer) { + } + + char* FastInt32ToBufferLeft(int32 i, char* buffer) { +- uint32 u = i; ++ uint32 u = 0; + if (i < 0) { + *buffer++ = '-'; +- u = -i; ++ u -= i; ++ } else { ++ u = i; + } + return FastUInt32ToBufferLeft(u, buffer); + } + diff --git a/BUILD b/BUILD index 6665de94..55f28582 100644 --- a/BUILD diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 311048a98f90..26690aa2a57b 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -507,7 +507,7 @@ def _com_google_protobuf(): # The patch includes # https://github.com/protocolbuffers/protobuf/pull/6333 and also uses # foreign_cc build for zlib as its dependency. - # TODO(asraa): remove this when > protobuf 3.8.0 is released. + # TODO(asraa): remove this when protobuf 3.10 is released. patch_args = ["-p1"], patches = ["@envoy//bazel:protobuf.patch"], ) @@ -521,7 +521,7 @@ def _com_google_protobuf(): # The patch includes # https://github.com/protocolbuffers/protobuf/pull/6333 and also uses # foreign_cc build for zlib as its dependency. - # TODO(asraa): remove this when > protobuf 3.8.0 is released. + # TODO(asraa): remove this when protobuf 3.10 is released. patch_args = ["-p1"], patches = ["@envoy//bazel:protobuf.patch"], ) diff --git a/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5650952886943744 b/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5650952886943744 new file mode 100644 index 000000000000..dbae8654ccbe --- /dev/null +++ b/test/common/router/route_corpus/clusterfuzz-testcase-minimized-route_fuzz_test-5650952886943744 @@ -0,0 +1,11 @@ +config { + vhds { + config_source { + api_config_source { + request_timeout { + nanos: -2147483648 + } + } + } + } +} From 3a4e276dc3da43010d9ceb846c9b7100889448cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vicent=20Mart=C3=AD?= Date: Mon, 30 Sep 2019 12:31:52 +0200 Subject: [PATCH 21/32] lua: allow setting complex values in dynamicMetadata (#8306) Description: This PR fixes a small TODO left by @dio: `Allow to set dynamic metadata using a table.` in the Lua API. Although the `get` function will convert any `Protobuf::Value` into its corresponding Lua type, the reverse `set` function would only allow plain strings to be passed as metadata. The code turns out to be rather straightforward. The most controversial bit is the heuristic to detect whether a Lua table should be converted to a Protobuf `Struct` or `ListValue`. This heuristic is the one that most Lua JSON implementations use to decide whether a table is serialized as a JSON map or array. Risk Level: Low Testing: Unit tests for complex Lua structures & failure cases have been added Docs Changes: The `lua_filter.rst` documentation has been updated to point out that other values besides strings can be passed as metadata. Release Notes: Updated Signed-off-by: Vicent Marti --- .../http/http_filters/lua_filter.rst | 20 +++- docs/root/intro/version_history.rst | 1 + .../extensions/filters/common/lua/wrappers.cc | 93 +++++++++++++++++++ .../extensions/filters/common/lua/wrappers.h | 6 ++ .../extensions/filters/http/lua/wrappers.cc | 14 ++- .../filters/http/lua/lua_filter_test.cc | 12 +++ .../filters/http/lua/wrappers_test.cc | 75 +++++++++++++++ 7 files changed, 217 insertions(+), 4 deletions(-) diff --git a/docs/root/configuration/http/http_filters/lua_filter.rst b/docs/root/configuration/http/http_filters/lua_filter.rst index 202bf74340e6..74192197e87a 100644 --- a/docs/root/configuration/http/http_filters/lua_filter.rst +++ b/docs/root/configuration/http/http_filters/lua_filter.rst @@ -504,7 +504,25 @@ set() dynamicMetadata:set(filterName, key, value) Sets key-value pair of a *filterName*'s metadata. *filterName* is a key specifying the target filter name, -e.g. *envoy.lb*. The type of *key* and *value* is *string*. +e.g. *envoy.lb*. The type of *key* is *string*. The type of *value* is any Lua type that can be mapped +to a metadata value: *table*, *numeric*, *boolean*, *string* and *nil*. When using a *table* as an argument, +its keys can only be *string* or *numeric*. + +.. code-block:: lua + + function envoy_on_request(request_handle) + local headers = request_handle:headers() + request_handle:streamInfo():dynamicMetadata():set("envoy.lua", "request.info", { + auth: headers:get("authorization), + token: headers:get("x-request-token"), + }) + end + + function envoy_on_response(response_handle) + local meta = response_handle:streamInfo():dynamicMetadata()["request.info"] + response_handle:logInfo("Auth: "..meta.auth..", token: "..meta.token) + end + __pairs() ^^^^^^^^^ diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index f43b08d85d9c..88005526b9cb 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -40,6 +40,7 @@ Version history * listeners: added :ref:`continue_on_listener_filters_timeout ` to configure whether a listener will still create a connection when listener filters time out. * listeners: added :ref:`HTTP inspector listener filter `. * lua: extended `httpCall()` and `respond()` APIs to accept headers with entry values that can be a string or table of strings. +* lua: extended `dynamicMetadata:set()` to allow setting complex values * metrics_service: added support for flushing histogram buckets. * outlier_detector: added :ref:`support for the grpc-status response header ` by mapping it to HTTP status. Guarded by envoy.reloadable_features.outlier_detection_support_for_grpc_status which defaults to true. * performance: new buffer implementation enabled by default (to disable add "--use-libevent-buffers 1" to the command-line arguments when starting Envoy). diff --git a/source/extensions/filters/common/lua/wrappers.cc b/source/extensions/filters/common/lua/wrappers.cc index 7b49f926a246..2e8d5d16c76d 100644 --- a/source/extensions/filters/common/lua/wrappers.cc +++ b/source/extensions/filters/common/lua/wrappers.cc @@ -87,6 +87,99 @@ void MetadataMapHelper::createTable(lua_State* state, } } +/** + * Converts the value on top of the Lua stack into a ProtobufWkt::Value. + * Any Lua types that cannot be directly mapped to Value types will + * yield an error. + */ +ProtobufWkt::Value MetadataMapHelper::loadValue(lua_State* state) { + ProtobufWkt::Value value; + int type = lua_type(state, -1); + + switch (type) { + case LUA_TNIL: + value.set_null_value(ProtobufWkt::NullValue()); + break; + case LUA_TNUMBER: + value.set_number_value(static_cast(lua_tonumber(state, -1))); + break; + case LUA_TBOOLEAN: + value.set_bool_value(lua_toboolean(state, -1) != 0); + break; + case LUA_TTABLE: { + int length = MetadataMapHelper::tableLength(state); + if (length > 0) { + *value.mutable_list_value() = MetadataMapHelper::loadList(state, length); + } else { + *value.mutable_struct_value() = MetadataMapHelper::loadStruct(state); + } + break; + } + case LUA_TSTRING: + value.set_string_value(lua_tostring(state, -1)); + break; + default: + luaL_error(state, "unexpected type '%s' in dynamicMetadata", lua_typename(state, type)); + } + + return value; +} + +/** + * Returns the length of a Lua table if it's actually shaped like a List, + * i.e. if all the keys are consecutive number values. Otherwise, returns -1. + */ +int MetadataMapHelper::tableLength(lua_State* state) { + double max = 0; + + lua_pushnil(state); + while (lua_next(state, -2) != 0) { + if (lua_type(state, -2) == LUA_TNUMBER) { + double k = lua_tonumber(state, -2); + if (floor(k) == k && k >= 1) { + if (k > max) { + max = k; + } + lua_pop(state, 1); + continue; + } + } + lua_pop(state, 2); + return -1; + } + return static_cast(max); +} + +ProtobufWkt::ListValue MetadataMapHelper::loadList(lua_State* state, int length) { + ProtobufWkt::ListValue list; + + for (int i = 1; i <= length; i++) { + lua_rawgeti(state, -1, i); + *list.add_values() = MetadataMapHelper::loadValue(state); + lua_pop(state, 1); + } + + return list; +} + +ProtobufWkt::Struct MetadataMapHelper::loadStruct(lua_State* state) { + ProtobufWkt::Struct struct_obj; + + lua_pushnil(state); + while (lua_next(state, -2) != 0) { + int key_type = lua_type(state, -2); + if (key_type != LUA_TSTRING) { + luaL_error(state, "unexpected type %s in table key (only string keys are supported)", + lua_typename(state, key_type)); + } + const char* key = lua_tostring(state, -2); + (*struct_obj.mutable_fields())[key] = MetadataMapHelper::loadValue(state); + lua_pop(state, 1); + } + + return struct_obj; +} + MetadataMapIterator::MetadataMapIterator(MetadataMapWrapper& parent) : parent_{parent}, current_{parent.metadata_.fields().begin()} {} diff --git a/source/extensions/filters/common/lua/wrappers.h b/source/extensions/filters/common/lua/wrappers.h index 3cdff2298d02..92aa697cfd73 100644 --- a/source/extensions/filters/common/lua/wrappers.h +++ b/source/extensions/filters/common/lua/wrappers.h @@ -46,6 +46,12 @@ struct MetadataMapHelper { static void setValue(lua_State* state, const ProtobufWkt::Value& value); static void createTable(lua_State* state, const Protobuf::Map& fields); + static ProtobufWkt::Value loadValue(lua_State* state); + +private: + static ProtobufWkt::Struct loadStruct(lua_State* state); + static ProtobufWkt::ListValue loadList(lua_State* state, int length); + static int tableLength(lua_State* state); }; /** diff --git a/source/extensions/filters/http/lua/wrappers.cc b/source/extensions/filters/http/lua/wrappers.cc index d17f02d6a0be..4874a716cec9 100644 --- a/source/extensions/filters/http/lua/wrappers.cc +++ b/source/extensions/filters/http/lua/wrappers.cc @@ -156,11 +156,19 @@ int DynamicMetadataMapWrapper::luaSet(lua_State* state) { luaL_error(state, "dynamic metadata map cannot be modified while iterating"); } - // TODO(dio): Allow to set dynamic metadata using a table. const char* filter_name = luaL_checkstring(state, 2); const char* key = luaL_checkstring(state, 3); - const char* value = luaL_checkstring(state, 4); - streamInfo().setDynamicMetadata(filter_name, MessageUtil::keyValueStruct(key, value)); + + // MetadataMapHelper::loadValue will convert the value on top of the Lua stack, + // so push a copy of the 3rd arg ("value") to the top. + lua_pushvalue(state, 4); + + ProtobufWkt::Struct value; + (*value.mutable_fields())[key] = Filters::Common::Lua::MetadataMapHelper::loadValue(state); + streamInfo().setDynamicMetadata(filter_name, value); + + // Pop the copy of the metadata value from the stack. + lua_pop(state, 1); return 0; } diff --git a/test/extensions/filters/http/lua/lua_filter_test.cc b/test/extensions/filters/http/lua/lua_filter_test.cc index a6bdafe76e14..5d96c613a877 100644 --- a/test/extensions/filters/http/lua/lua_filter_test.cc +++ b/test/extensions/filters/http/lua/lua_filter_test.cc @@ -1546,7 +1546,9 @@ TEST_F(LuaHttpFilterTest, SetGetDynamicMetadata) { const std::string SCRIPT{R"EOF( function envoy_on_request(request_handle) request_handle:streamInfo():dynamicMetadata():set("envoy.lb", "foo", "bar") + request_handle:streamInfo():dynamicMetadata():set("envoy.lb", "complex", {x="abcd", y=1234}) request_handle:logTrace(request_handle:streamInfo():dynamicMetadata():get("envoy.lb")["foo"]) + request_handle:logTrace(request_handle:streamInfo():dynamicMetadata():get("envoy.lb")["complex"].x) end )EOF"}; @@ -1559,6 +1561,7 @@ TEST_F(LuaHttpFilterTest, SetGetDynamicMetadata) { EXPECT_EQ(0, stream_info.dynamicMetadata().filter_metadata_size()); EXPECT_CALL(decoder_callbacks_, streamInfo()).WillOnce(ReturnRef(stream_info)); EXPECT_CALL(*filter_, scriptLog(spdlog::level::trace, StrEq("bar"))); + EXPECT_CALL(*filter_, scriptLog(spdlog::level::trace, StrEq("abcd"))); EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, true)); EXPECT_EQ(1, stream_info.dynamicMetadata().filter_metadata_size()); EXPECT_EQ("bar", stream_info.dynamicMetadata() @@ -1567,6 +1570,15 @@ TEST_F(LuaHttpFilterTest, SetGetDynamicMetadata) { .fields() .at("foo") .string_value()); + + const ProtobufWkt::Struct& meta_complex = stream_info.dynamicMetadata() + .filter_metadata() + .at("envoy.lb") + .fields() + .at("complex") + .struct_value(); + EXPECT_EQ("abcd", meta_complex.fields().at("x").string_value()); + EXPECT_EQ(1234.0, meta_complex.fields().at("y").number_value()); } // Check the connection. diff --git a/test/extensions/filters/http/lua/wrappers_test.cc b/test/extensions/filters/http/lua/wrappers_test.cc index 3b934e4981fe..2ce294f854d1 100644 --- a/test/extensions/filters/http/lua/wrappers_test.cc +++ b/test/extensions/filters/http/lua/wrappers_test.cc @@ -311,6 +311,81 @@ TEST_F(LuaStreamInfoWrapperTest, SetGetAndIterateDynamicMetadata) { wrapper.reset(); } +// Set, get complex key/values in stream info dynamic metadata. +TEST_F(LuaStreamInfoWrapperTest, SetGetComplexDynamicMetadata) { + const std::string SCRIPT{R"EOF( + function callMe(object) + object:dynamicMetadata():set("envoy.lb", "foo", {x=1234, y="baz", z=true}) + object:dynamicMetadata():set("envoy.lb", "so", {"cool", "and", "dynamic", true}) + + testPrint(tostring(object:dynamicMetadata():get("envoy.lb")["foo"].x)) + testPrint(object:dynamicMetadata():get("envoy.lb")["foo"].y) + testPrint(tostring(object:dynamicMetadata():get("envoy.lb")["foo"].z)) + testPrint(object:dynamicMetadata():get("envoy.lb")["so"][1]) + testPrint(object:dynamicMetadata():get("envoy.lb")["so"][2]) + testPrint(object:dynamicMetadata():get("envoy.lb")["so"][3]) + testPrint(tostring(object:dynamicMetadata():get("envoy.lb")["so"][4])) + end + )EOF"}; + + InSequence s; + setup(SCRIPT); + + StreamInfo::StreamInfoImpl stream_info(Http::Protocol::Http2, test_time_.timeSystem()); + EXPECT_EQ(0, stream_info.dynamicMetadata().filter_metadata_size()); + Filters::Common::Lua::LuaDeathRef wrapper( + StreamInfoWrapper::create(coroutine_->luaState(), stream_info), true); + EXPECT_CALL(*this, testPrint("1234")); + EXPECT_CALL(*this, testPrint("baz")); + EXPECT_CALL(*this, testPrint("true")); + EXPECT_CALL(*this, testPrint("cool")); + EXPECT_CALL(*this, testPrint("and")); + EXPECT_CALL(*this, testPrint("dynamic")); + EXPECT_CALL(*this, testPrint("true")); + start("callMe"); + + EXPECT_EQ(1, stream_info.dynamicMetadata().filter_metadata_size()); + const ProtobufWkt::Struct& meta_foo = stream_info.dynamicMetadata() + .filter_metadata() + .at("envoy.lb") + .fields() + .at("foo") + .struct_value(); + + EXPECT_EQ(1234.0, meta_foo.fields().at("x").number_value()); + EXPECT_EQ("baz", meta_foo.fields().at("y").string_value()); + EXPECT_EQ(true, meta_foo.fields().at("z").bool_value()); + + const ProtobufWkt::ListValue& meta_so = + stream_info.dynamicMetadata().filter_metadata().at("envoy.lb").fields().at("so").list_value(); + + EXPECT_EQ(4, meta_so.values_size()); + EXPECT_EQ("cool", meta_so.values(0).string_value()); + EXPECT_EQ("and", meta_so.values(1).string_value()); + EXPECT_EQ("dynamic", meta_so.values(2).string_value()); + EXPECT_EQ(true, meta_so.values(3).bool_value()); + + wrapper.reset(); +} + +// Bad types in table +TEST_F(LuaStreamInfoWrapperTest, BadTypesInTableForDynamicMetadata) { + const std::string SCRIPT{R"EOF( + function callMe(object) + object:dynamicMetadata():set("envoy.lb", "hello", {x="world", y=function(a, b) end}) + end + )EOF"}; + + InSequence s; + setup(SCRIPT); + + StreamInfo::StreamInfoImpl stream_info(Http::Protocol::Http2, test_time_.timeSystem()); + Filters::Common::Lua::LuaDeathRef wrapper( + StreamInfoWrapper::create(coroutine_->luaState(), stream_info), true); + EXPECT_THROW_WITH_MESSAGE(start("callMe"), Filters::Common::Lua::LuaException, + "[string \"...\"]:3: unexpected type 'function' in dynamicMetadata"); +} + // Modify during iteration. TEST_F(LuaStreamInfoWrapperTest, ModifyDuringIterationForDynamicMetadata) { const std::string SCRIPT{R"EOF( From d76e9b390082828e9b9047a71c3eb96faca8be0d Mon Sep 17 00:00:00 2001 From: Henry Yang <4411287+HenryYYang@users.noreply.github.com> Date: Mon, 30 Sep 2019 13:46:48 -0700 Subject: [PATCH 22/32] Enforce hashtagging for Redis clusters (#8418) Signed-off-by: Henry Yang --- docs/root/intro/version_history.rst | 1 + .../clusters/redis/redis_cluster_lb.cc | 6 ++--- .../clusters/redis/redis_cluster_lb.h | 14 ++++++++++- .../network/redis_proxy/conn_pool_impl.cc | 6 ++--- .../clusters/redis/redis_cluster_lb_test.cc | 23 +++++++++++++++++++ 5 files changed, 43 insertions(+), 7 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 88005526b9cb..3a7afd519c07 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -49,6 +49,7 @@ Version history * redis: added :ref:`enable_command_stats ` to enable :ref:`per command statistics ` for upstream clusters. * redis: added :ref:`read_policy ` to allow reading from redis replicas for Redis Cluster deployments. * redis: fix a bug where the redis health checker ignored the upstream auth password. +* redis: enable_hashtaging is always enabled when the upstream uses open source Redis cluster protocol. * regex: introduce new :ref:`RegexMatcher ` type that provides a safe regex implementation for untrusted user input. This type is now used in all configuration that processes user provided input. See :ref:`deprecated configuration details diff --git a/source/extensions/clusters/redis/redis_cluster_lb.cc b/source/extensions/clusters/redis/redis_cluster_lb.cc index b4f3b1d06a2c..442602e8529d 100644 --- a/source/extensions/clusters/redis/redis_cluster_lb.cc +++ b/source/extensions/clusters/redis/redis_cluster_lb.cc @@ -176,11 +176,11 @@ bool isReadRequest(const NetworkFilters::Common::Redis::RespValue& request) { } // namespace RedisLoadBalancerContextImpl::RedisLoadBalancerContextImpl( - const std::string& key, bool enabled_hashtagging, bool use_crc16, + const std::string& key, bool enabled_hashtagging, bool is_redis_cluster, const NetworkFilters::Common::Redis::RespValue& request, NetworkFilters::Common::Redis::Client::ReadPolicy read_policy) - : hash_key_(use_crc16 ? Crc16::crc16(hashtag(key, enabled_hashtagging)) - : MurmurHash::murmurHash2_64(hashtag(key, enabled_hashtagging))), + : hash_key_(is_redis_cluster ? Crc16::crc16(hashtag(key, true)) + : MurmurHash::murmurHash2_64(hashtag(key, enabled_hashtagging))), is_read_(isReadRequest(request)), read_policy_(read_policy) {} // Inspired by the redis-cluster hashtagging algorithm diff --git a/source/extensions/clusters/redis/redis_cluster_lb.h b/source/extensions/clusters/redis/redis_cluster_lb.h index 3ad30cf76074..1ee23c11ed18 100644 --- a/source/extensions/clusters/redis/redis_cluster_lb.h +++ b/source/extensions/clusters/redis/redis_cluster_lb.h @@ -67,7 +67,19 @@ class RedisLoadBalancerContext { class RedisLoadBalancerContextImpl : public RedisLoadBalancerContext, public Upstream::LoadBalancerContextBase { public: - RedisLoadBalancerContextImpl(const std::string& key, bool enabled_hashtagging, bool use_crc16, + /** + * The load balancer context for Redis requests. Note that is_redis_cluster implies using Redis + * cluster which require us to always enable hashtagging. + * @param key specify the key for the Redis request. + * @param enabled_hashtagging specify whether to enable hashtagging, this will always be true if + * is_redis_cluster is true. + * @param is_redis_cluster specify whether this is a request for redis cluster, if true the key + * will be hashed using crc16. + * @param request specify the Redis request. + * @param read_policy specify the read policy. + */ + RedisLoadBalancerContextImpl(const std::string& key, bool enabled_hashtagging, + bool is_redis_cluster, const NetworkFilters::Common::Redis::RespValue& request, NetworkFilters::Common::Redis::Client::ReadPolicy read_policy = NetworkFilters::Common::Redis::Client::ReadPolicy::Master); diff --git a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc index 7304ef902325..24c8caf7a2ab 100644 --- a/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc +++ b/source/extensions/filters/network/redis_proxy/conn_pool_impl.cc @@ -208,9 +208,9 @@ InstanceImpl::ThreadLocalPool::makeRequest(const std::string& key, return nullptr; } - const bool use_crc16 = is_redis_cluster_; - Clusters::Redis::RedisLoadBalancerContextImpl lb_context( - key, parent_.config_.enableHashtagging(), use_crc16, request, parent_.config_.readPolicy()); + Clusters::Redis::RedisLoadBalancerContextImpl lb_context(key, parent_.config_.enableHashtagging(), + is_redis_cluster_, request, + parent_.config_.readPolicy()); Upstream::HostConstSharedPtr host = cluster_->loadBalancer().chooseHost(&lb_context); if (!host) { return nullptr; diff --git a/test/extensions/clusters/redis/redis_cluster_lb_test.cc b/test/extensions/clusters/redis/redis_cluster_lb_test.cc index 5e0ae78de9b2..8e0d8214966b 100644 --- a/test/extensions/clusters/redis/redis_cluster_lb_test.cc +++ b/test/extensions/clusters/redis/redis_cluster_lb_test.cc @@ -423,6 +423,29 @@ TEST_F(RedisLoadBalancerContextImplTest, UnsupportedCommand) { EXPECT_EQ(NetworkFilters::Common::Redis::Client::ReadPolicy::Master, context3.readPolicy()); } +TEST_F(RedisLoadBalancerContextImplTest, EnforceHashTag) { + std::vector set_foo(3); + set_foo[0].type(NetworkFilters::Common::Redis::RespType::BulkString); + set_foo[0].asString() = "set"; + set_foo[1].type(NetworkFilters::Common::Redis::RespType::BulkString); + set_foo[1].asString() = "{foo}bar"; + set_foo[2].type(NetworkFilters::Common::Redis::RespType::BulkString); + set_foo[2].asString() = "bar"; + + NetworkFilters::Common::Redis::RespValue set_request; + set_request.type(NetworkFilters::Common::Redis::RespType::Array); + set_request.asArray().swap(set_foo); + + // Enable_hash tagging should be override when is_redis_cluster is true. This is treated like + // "foo" + RedisLoadBalancerContextImpl context2("{foo}bar", false, true, set_request, + NetworkFilters::Common::Redis::Client::ReadPolicy::Master); + + EXPECT_EQ(absl::optional(44950), context2.computeHashKey()); + EXPECT_EQ(false, context2.isReadCommand()); + EXPECT_EQ(NetworkFilters::Common::Redis::Client::ReadPolicy::Master, context2.readPolicy()); +} + } // namespace Redis } // namespace Clusters } // namespace Extensions From b72f38a1770b9d18afba6d65ac229e308ada4341 Mon Sep 17 00:00:00 2001 From: Yi Tang Date: Tue, 1 Oct 2019 14:55:44 +0800 Subject: [PATCH 23/32] tools: proto sync shebang line for portability (#8430) Description: use /usr/bin/env python3 for portability in proto_sync.py. Risk Level: low Signed-off-by: Yi Tang --- tools/proto_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/proto_sync.py b/tools/proto_sync.py index 4911ac767215..809126eff520 100755 --- a/tools/proto_sync.py +++ b/tools/proto_sync.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Diff or copy protoxform artifacts from Bazel cache back to the source tree. From 27c77491d676d98bf6ab7eb539ad6662034d8ae0 Mon Sep 17 00:00:00 2001 From: Joshua Marantz Date: Tue, 1 Oct 2019 08:18:15 -0400 Subject: [PATCH 24/32] stats: Recent lookups implementation (#8376) * recent-lookups class impl & tests. Signed-off-by: Joshua Marantz --- source/common/stats/BUILD | 9 ++ source/common/stats/recent_lookups.cc | 60 ++++++++++ source/common/stats/recent_lookups.h | 77 +++++++++++++ test/common/stats/BUILD | 24 ++++ .../common/stats/recent_lookups_speed_test.cc | 82 +++++++++++++ test/common/stats/recent_lookups_test.cc | 108 ++++++++++++++++++ 6 files changed, 360 insertions(+) create mode 100644 source/common/stats/recent_lookups.cc create mode 100644 source/common/stats/recent_lookups.h create mode 100644 test/common/stats/recent_lookups_speed_test.cc create mode 100644 test/common/stats/recent_lookups_test.cc diff --git a/source/common/stats/BUILD b/source/common/stats/BUILD index ed5efa052b47..224c006fc395 100644 --- a/source/common/stats/BUILD +++ b/source/common/stats/BUILD @@ -87,6 +87,15 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "recent_lookups_lib", + srcs = ["recent_lookups.cc"], + hdrs = ["recent_lookups.h"], + deps = [ + "//source/common/common:assert_lib", + ], +) + envoy_cc_library( name = "store_impl_lib", hdrs = ["store_impl.h"], diff --git a/source/common/stats/recent_lookups.cc b/source/common/stats/recent_lookups.cc new file mode 100644 index 000000000000..994441cccb02 --- /dev/null +++ b/source/common/stats/recent_lookups.cc @@ -0,0 +1,60 @@ +#include "common/stats/recent_lookups.h" + +#include "common/common/assert.h" + +namespace Envoy { +namespace Stats { + +void RecentLookups::lookup(absl::string_view str) { + ++total_; + if (capacity_ == 0) { + return; + } + auto map_iter = map_.find(str); + if (map_iter != map_.end()) { + // The item is already in the list. Bump its reference-count and move it to + // the front of the list. + auto list_iter = map_iter->second; + ++list_iter->count_; + if (list_iter != list_.begin()) { + list_.splice(list_.begin(), list_, list_iter); + } + } else { + ASSERT(list_.size() <= capacity_); + // Evict oldest item if needed. + if (list_.size() >= capacity_) { + evictOne(); + } + + // The string storage is in the list entry. + list_.push_front(ItemCount{std::string(str), 1}); + auto list_iter = list_.begin(); + map_[list_iter->item_] = list_iter; + } + ASSERT(list_.size() == map_.size()); +} + +void RecentLookups::forEach(const IterFn& fn) const { + for (const ItemCount& item_count : list_) { + fn(item_count.item_, item_count.count_); + } +} + +void RecentLookups::setCapacity(uint64_t capacity) { + capacity_ = capacity; + while (capacity_ < list_.size()) { + evictOne(); + } +} + +void RecentLookups::evictOne() { + ASSERT(!list_.empty()); + ASSERT(!map_.empty()); + const ItemCount& item_count = list_.back(); + int erased = map_.erase(item_count.item_); + ASSERT(erased == 1); + list_.pop_back(); +} + +} // namespace Stats +} // namespace Envoy diff --git a/source/common/stats/recent_lookups.h b/source/common/stats/recent_lookups.h new file mode 100644 index 000000000000..3e633bca8dbd --- /dev/null +++ b/source/common/stats/recent_lookups.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include + +#include "absl/container/flat_hash_map.h" +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Stats { + +// Remembers the last 'Capacity' items passed to lookup(). +class RecentLookups { +public: + /** + * Records a lookup of a string. Only the last 'Capacity' lookups are remembered. + * + * @param str the item being looked up. + */ + void lookup(absl::string_view str); + + using IterFn = std::function; + + /** + * Calls fn(item, count) for each of the remembered lookups. + * + * @param fn The function to call for every recently looked up item. + */ + void forEach(const IterFn& fn) const; + + /** + * @return the total number of lookups since tracking began. + */ + uint64_t total() const { return total_; } + + /** + * Clears out all contents. + */ + void clear() { + total_ = 0; + map_.clear(); + list_.clear(); + } + + /** + * Controls the maximum number of recent lookups to remember. If set to 0, + * then only lookup counts is tracked. + * @param capacity The number of lookups to remember. + */ + void setCapacity(uint64_t capacity); + + /** + * @return The configured capacity. + */ + uint64_t capacity() const { return capacity_; } + +private: + void evictOne(); + + struct ItemCount { + std::string item_; + uint64_t count_; + }; + using List = std::list; + List list_; + + // TODO(jmarantz): we could make this more compact by making this a set of + // list-iterators with heterogeneous hash/compare functors. + using Map = absl::flat_hash_map; + Map map_; + uint64_t total_{0}; + uint64_t capacity_{0}; +}; + +} // namespace Stats +} // namespace Envoy diff --git a/test/common/stats/BUILD b/test/common/stats/BUILD index b4b528194103..65ed7bf5fd45 100644 --- a/test/common/stats/BUILD +++ b/test/common/stats/BUILD @@ -39,6 +39,30 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "recent_lookups_test", + srcs = ["recent_lookups_test.cc"], + deps = [ + "//source/common/common:utility_lib", + "//source/common/stats:recent_lookups_lib", + "//test/test_common:logging_lib", + "//test/test_common:simulated_time_system_lib", + ], +) + +envoy_cc_test_binary( + name = "recent_lookups_speed_test", + srcs = ["recent_lookups_speed_test.cc"], + external_deps = [ + "benchmark", + ], + deps = [ + "//source/common/common:utility_lib", + "//source/common/runtime:runtime_lib", + "//source/common/stats:recent_lookups_lib", + ], +) + envoy_cc_test( name = "stat_merger_test", srcs = ["stat_merger_test.cc"], diff --git a/test/common/stats/recent_lookups_speed_test.cc b/test/common/stats/recent_lookups_speed_test.cc new file mode 100644 index 000000000000..15224a7c569d --- /dev/null +++ b/test/common/stats/recent_lookups_speed_test.cc @@ -0,0 +1,82 @@ +// Note: this should be run with --compilation_mode=opt, and would benefit from a +// quiescent system with disabled cstate power management. +// +// NOLINT(namespace-envoy) +// +// Running bazel-bin/test/common/stats/recent_lookups_speed_test +// Run on (12 X 4500 MHz CPU s) +// CPU Caches: +// L1 Data 32K (x6) +// L1 Instruction 32K (x6) +// L2 Unified 1024K (x6) +// L3 Unified 8448K (x1) +// Load Average: 1.32, 7.40, 10.21 +// ***WARNING*** CPU scaling is enabled, the benchmark real time measurements may be noisy and will +// incur extra overhead. +// ----------------------------------------------------------------- +// Benchmark Time CPU Iterations +// ----------------------------------------------------------------- +// BM_LookupsMixed 87068 ns 87068 ns 6955 +// BM_LookupsNoEvictions 45662 ns 45662 ns 15329 +// BM_LookupsAllEvictions 83015 ns 83015 ns 8435 + +#include "common/runtime/runtime_impl.h" +#include "common/stats/recent_lookups.h" + +#include "absl/strings/str_cat.h" +#include "benchmark/benchmark.h" + +class RecentLookupsSpeedTest { +public: + RecentLookupsSpeedTest(uint64_t lookup_variants, uint64_t capacity) { + recent_lookups_.setCapacity(capacity); + Envoy::Runtime::RandomGeneratorImpl random; + lookups_.reserve(lookup_variants); + for (size_t i = 0; i < lookup_variants; ++i) { + lookups_.push_back(absl::StrCat("lookup #", random.random())); + } + } + + void test(benchmark::State& state) { + for (auto _ : state) { + Envoy::Runtime::RandomGeneratorImpl random; + for (uint64_t i = 0; i < lookups_.size(); ++i) { + recent_lookups_.lookup(lookups_[random.random() % lookups_.size()]); + } + } + } + +private: + std::vector lookups_; + Envoy::Stats::RecentLookups recent_lookups_; +}; + +static void BM_LookupsMixed(benchmark::State& state) { + RecentLookupsSpeedTest speed_test(1000, 500); + speed_test.test(state); +} +BENCHMARK(BM_LookupsMixed); + +static void BM_LookupsNoEvictions(benchmark::State& state) { + RecentLookupsSpeedTest speed_test(1000, 1000); + speed_test.test(state); +} +BENCHMARK(BM_LookupsNoEvictions); + +static void BM_LookupsAllEvictions(benchmark::State& state) { + RecentLookupsSpeedTest speed_test(1000, 10); + speed_test.test(state); +} +BENCHMARK(BM_LookupsAllEvictions); + +int main(int argc, char** argv) { + Envoy::Thread::MutexBasicLockable lock; + Envoy::Logger::Context logger_context(spdlog::level::warn, + Envoy::Logger::Logger::DEFAULT_LOG_FORMAT, lock); + benchmark::Initialize(&argc, argv); + + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::RunSpecifiedBenchmarks(); +} diff --git a/test/common/stats/recent_lookups_test.cc b/test/common/stats/recent_lookups_test.cc new file mode 100644 index 000000000000..e8be8f9d90be --- /dev/null +++ b/test/common/stats/recent_lookups_test.cc @@ -0,0 +1,108 @@ +#include +#include + +#include "common/common/utility.h" +#include "common/stats/recent_lookups.h" + +#include "test/test_common/logging.h" + +#include "absl/strings/str_cat.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Stats { +namespace { + +class RecentLookupsTest : public testing::Test { +protected: + std::string joinLookups() { + using ItemCount = std::pair; + std::vector items; + recent_lookups_.forEach([&items](absl::string_view item, uint64_t count) { + items.emplace_back(ItemCount(std::string(item), count)); + }); + std::sort(items.begin(), items.end(), [](const ItemCount& a, const ItemCount& b) -> bool { + if (a.second == b.second) { + return a.first < b.first; + } + return a.second < b.second; + }); + std::vector accum; + accum.reserve(items.size()); + for (const auto& item : items) { + accum.push_back(absl::StrCat(item.second, ": ", item.first)); + } + return StringUtil::join(accum, " "); + } + + RecentLookups recent_lookups_; +}; + +TEST_F(RecentLookupsTest, Empty) { EXPECT_EQ("", joinLookups()); } + +TEST_F(RecentLookupsTest, One) { + recent_lookups_.lookup("Hello"); + EXPECT_EQ("", joinLookups()); + recent_lookups_.setCapacity(10); + EXPECT_EQ(1, recent_lookups_.total()); + recent_lookups_.lookup("Hello"); + EXPECT_EQ(2, recent_lookups_.total()); + EXPECT_EQ("1: Hello", joinLookups()); + + recent_lookups_.clear(); + EXPECT_EQ("", joinLookups()); + EXPECT_EQ(0, recent_lookups_.total()); + recent_lookups_.lookup("Hello"); + EXPECT_EQ(1, recent_lookups_.total()); + EXPECT_EQ("1: Hello", joinLookups()); + recent_lookups_.setCapacity(0); + EXPECT_EQ("", joinLookups()); + EXPECT_EQ(1, recent_lookups_.total()); +} + +TEST_F(RecentLookupsTest, DropOne) { + recent_lookups_.setCapacity(10); + for (int i = 0; i < 11; ++i) { + recent_lookups_.lookup(absl::StrCat("lookup", i)); + } + EXPECT_EQ("1: lookup1 " + "1: lookup10 " + "1: lookup2 " + "1: lookup3 " + "1: lookup4 " + "1: lookup5 " + "1: lookup6 " + "1: lookup7 " + "1: lookup8 " + "1: lookup9", + joinLookups()); + recent_lookups_.clear(); + EXPECT_EQ("", joinLookups()); +} + +TEST_F(RecentLookupsTest, RepeatDrop) { + recent_lookups_.setCapacity(10); + recent_lookups_.lookup("drop_early"); + for (int i = 0; i < 11; ++i) { + recent_lookups_.lookup(absl::StrCat("lookup", i)); + recent_lookups_.lookup(absl::StrCat("lookup", i)); + } + recent_lookups_.lookup("add_late"); + EXPECT_EQ("1: add_late " + "2: lookup10 " + "2: lookup2 " + "2: lookup3 " + "2: lookup4 " + "2: lookup5 " + "2: lookup6 " + "2: lookup7 " + "2: lookup8 " + "2: lookup9", + joinLookups()); + recent_lookups_.clear(); + EXPECT_EQ("", joinLookups()); +} + +} // namespace +} // namespace Stats +} // namespace Envoy From a119a0703de8de2b7351331aa502879a7e1044d7 Mon Sep 17 00:00:00 2001 From: Tony Allen Date: Tue, 1 Oct 2019 10:32:16 -0700 Subject: [PATCH 25/32] adaptive concurrency: Filter configuration and integration tests (#8244) Signed-off-by: Tony Allen --- source/extensions/extensions_build_config.bzl | 4 +- .../filters/http/adaptive_concurrency/BUILD | 14 ++ .../adaptive_concurrency_filter.cc | 10 +- .../gradient_controller.cc | 22 ++- .../gradient_controller.h | 14 +- .../http/adaptive_concurrency/config.cc | 49 ++++++ .../http/adaptive_concurrency/config.h | 32 ++++ test/common/protobuf/utility_test.cc | 10 +- .../filters/http/adaptive_concurrency/BUILD | 15 ++ ...ive_concurrency_filter_integration_test.cc | 162 ++++++++++++++++++ ...tive_concurrency_filter_integration_test.h | 80 +++++++++ .../gradient_controller_test.cc | 6 +- 12 files changed, 388 insertions(+), 30 deletions(-) create mode 100644 source/extensions/filters/http/adaptive_concurrency/config.cc create mode 100644 source/extensions/filters/http/adaptive_concurrency/config.h create mode 100644 test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.cc create mode 100644 test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.h diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 5366f7ba55fb..fd78ee5b7811 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -31,9 +31,7 @@ EXTENSIONS = { # HTTP filters # - # NOTE: The adaptive concurrency filter does not have a proper filter - # implemented right now. We are just referencing the filter lib here. - "envoy.filters.http.adaptive_concurrency": "//source/extensions/filters/http/adaptive_concurrency:adaptive_concurrency_filter_lib", + "envoy.filters.http.adaptive_concurrency": "//source/extensions/filters/http/adaptive_concurrency:config", "envoy.filters.http.buffer": "//source/extensions/filters/http/buffer:config", "envoy.filters.http.cors": "//source/extensions/filters/http/cors:config", "envoy.filters.http.csrf": "//source/extensions/filters/http/csrf:config", diff --git a/source/extensions/filters/http/adaptive_concurrency/BUILD b/source/extensions/filters/http/adaptive_concurrency/BUILD index 5d86773ebf8c..cc73c3ce50d0 100644 --- a/source/extensions/filters/http/adaptive_concurrency/BUILD +++ b/source/extensions/filters/http/adaptive_concurrency/BUILD @@ -24,3 +24,17 @@ envoy_cc_library( "@envoy_api//envoy/config/filter/http/adaptive_concurrency/v2alpha:adaptive_concurrency_cc", ], ) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + "//include/envoy/registry", + "//source/common/config:filter_json_lib", + "//source/extensions/filters/http:well_known_names", + "//source/extensions/filters/http/adaptive_concurrency:adaptive_concurrency_filter_lib", + "//source/extensions/filters/http/adaptive_concurrency/concurrency_controller:concurrency_controller_lib", + "//source/extensions/filters/http/common:factory_base_lib", + ], +) diff --git a/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc index 076ff9c57b60..41eaecee47e2 100644 --- a/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc +++ b/source/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter.cc @@ -26,9 +26,8 @@ AdaptiveConcurrencyFilter::AdaptiveConcurrencyFilter( Http::FilterHeadersStatus AdaptiveConcurrencyFilter::decodeHeaders(Http::HeaderMap&, bool) { if (controller_->forwardingDecision() == ConcurrencyController::RequestForwardingAction::Block) { - // TODO (tonya11en): Remove filler words. - decoder_callbacks_->sendLocalReply(Http::Code::ServiceUnavailable, "filler words", nullptr, - absl::nullopt, "more filler words"); + decoder_callbacks_->sendLocalReply(Http::Code::ServiceUnavailable, "", nullptr, absl::nullopt, + "reached concurrency limit"); return Http::FilterHeadersStatus::StopIteration; } @@ -46,10 +45,7 @@ Http::FilterHeadersStatus AdaptiveConcurrencyFilter::decodeHeaders(Http::HeaderM return Http::FilterHeadersStatus::Continue; } -void AdaptiveConcurrencyFilter::encodeComplete() { - ASSERT(deferred_sample_task_); - deferred_sample_task_.reset(); -} +void AdaptiveConcurrencyFilter::encodeComplete() { deferred_sample_task_.reset(); } void AdaptiveConcurrencyFilter::onDestroy() { if (deferred_sample_task_) { diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc index 86594e636083..7a8b0805da98 100644 --- a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.cc @@ -42,7 +42,7 @@ GradientControllerConfig::GradientControllerConfig( PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(proto_config, sample_aggregate_percentile, 50) / 100.0) {} -GradientController::GradientController(GradientControllerConfigSharedPtr config, +GradientController::GradientController(GradientControllerConfig config, Event::Dispatcher& dispatcher, Runtime::Loader&, const std::string& stats_prefix, Stats::Scope& scope, Runtime::RandomGenerator& random) @@ -65,16 +65,17 @@ GradientController::GradientController(GradientControllerConfigSharedPtr config, resetSampleWindow(); } - sample_reset_timer_->enableTimer(config_->sampleRTTCalcInterval()); + sample_reset_timer_->enableTimer(config_.sampleRTTCalcInterval()); }); - sample_reset_timer_->enableTimer(config_->sampleRTTCalcInterval()); + sample_reset_timer_->enableTimer(config_.sampleRTTCalcInterval()); stats_.concurrency_limit_.set(concurrency_limit_.load()); } GradientControllerStats GradientController::generateStats(Stats::Scope& scope, const std::string& stats_prefix) { - return {ALL_GRADIENT_CONTROLLER_STATS(POOL_GAUGE_PREFIX(scope, stats_prefix))}; + return {ALL_GRADIENT_CONTROLLER_STATS(POOL_COUNTER_PREFIX(scope, stats_prefix), + POOL_GAUGE_PREFIX(scope, stats_prefix))}; } void GradientController::enterMinRTTSamplingWindow() { @@ -104,7 +105,7 @@ void GradientController::updateMinRTT() { } min_rtt_calc_timer_->enableTimer( - applyJitter(config_->minRTTCalcInterval(), config_->jitterPercent())); + applyJitter(config_.minRTTCalcInterval(), config_.jitterPercent())); } std::chrono::milliseconds GradientController::applyJitter(std::chrono::milliseconds interval, @@ -126,11 +127,13 @@ void GradientController::resetSampleWindow() { } sample_rtt_ = processLatencySamplesAndClear(); + stats_.sample_rtt_msecs_.set( + std::chrono::duration_cast(sample_rtt_).count()); updateConcurrencyLimit(calculateNewLimit()); } std::chrono::microseconds GradientController::processLatencySamplesAndClear() { - const std::array quantile{config_->sampleAggregatePercentile()}; + const std::array quantile{config_.sampleAggregatePercentile()}; std::array calculated_quantile; hist_approx_quantile(latency_sample_hist_.get(), quantile.data(), 1, calculated_quantile.data()); hist_clear(latency_sample_hist_.get()); @@ -141,7 +144,7 @@ uint32_t GradientController::calculateNewLimit() { // Calculate the gradient value, ensuring it remains below the configured maximum. ASSERT(sample_rtt_.count() > 0); const double raw_gradient = static_cast(min_rtt_.count()) / sample_rtt_.count(); - const double gradient = std::min(config_->maxGradient(), raw_gradient); + const double gradient = std::min(config_.maxGradient(), raw_gradient); stats_.gradient_.set(gradient); const double limit = concurrencyLimit() * gradient; @@ -152,7 +155,7 @@ uint32_t GradientController::calculateNewLimit() { // in the range [1, configured_max]. const auto clamp = [](int min, int max, int val) { return std::max(min, std::min(max, val)); }; const uint32_t new_limit = limit + burst_headroom; - return clamp(1, config_->maxConcurrencyLimit(), new_limit); + return clamp(1, config_.maxConcurrencyLimit(), new_limit); } RequestForwardingAction GradientController::forwardingDecision() { @@ -167,6 +170,7 @@ RequestForwardingAction GradientController::forwardingDecision() { ++num_rq_outstanding_; return RequestForwardingAction::Forward; } + stats_.rq_blocked_.inc(); return RequestForwardingAction::Block; } @@ -183,7 +187,7 @@ void GradientController::recordLatencySample(std::chrono::nanoseconds rq_latency sample_count = hist_sample_count(latency_sample_hist_.get()); } - if (inMinRTTSamplingWindow() && sample_count >= config_->minRTTAggregateRequestCount()) { + if (inMinRTTSamplingWindow() && sample_count >= config_.minRTTAggregateRequestCount()) { // This sample has pushed the request count over the request count requirement for the minRTT // recalculation. It must now be finished. updateMinRTT(); diff --git a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h index 91566d887a39..2b6aa741095f 100644 --- a/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h +++ b/source/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller.h @@ -24,20 +24,22 @@ namespace ConcurrencyController { /** * All stats for the gradient controller. */ -#define ALL_GRADIENT_CONTROLLER_STATS(GAUGE) \ +#define ALL_GRADIENT_CONTROLLER_STATS(COUNTER, GAUGE) \ + COUNTER(rq_blocked) \ GAUGE(concurrency_limit, NeverImport) \ GAUGE(gradient, NeverImport) \ GAUGE(burst_queue_size, NeverImport) \ - GAUGE(min_rtt_msecs, NeverImport) + GAUGE(min_rtt_msecs, NeverImport) \ + GAUGE(sample_rtt_msecs, NeverImport) /** * Wrapper struct for gradient controller stats. @see stats_macros.h */ struct GradientControllerStats { - ALL_GRADIENT_CONTROLLER_STATS(GENERATE_GAUGE_STRUCT) + ALL_GRADIENT_CONTROLLER_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) }; -class GradientControllerConfig { +class GradientControllerConfig : public Logger::Loggable { public: GradientControllerConfig( const envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig& @@ -139,7 +141,7 @@ using GradientControllerConfigSharedPtr = std::shared_ptr controller; + using Proto = envoy::config::filter::http::adaptive_concurrency::v2alpha::AdaptiveConcurrency; + ASSERT(config.concurrency_controller_config_case() == + Proto::ConcurrencyControllerConfigCase::kGradientControllerConfig); + controller = std::make_shared( + config.gradient_controller_config(), context.dispatcher(), context.runtime(), + acc_stats_prefix + "gradient_controller.", context.scope(), context.random()); + + AdaptiveConcurrencyFilterConfigSharedPtr filter_config( + new AdaptiveConcurrencyFilterConfig(config, context.runtime(), std::move(acc_stats_prefix), + context.scope(), context.timeSource())); + + return [filter_config, controller](Http::FilterChainFactoryCallbacks& callbacks) -> void { + callbacks.addStreamFilter( + std::make_shared(filter_config, controller)); + }; +} + +/** + * Static registration for the adaptive_concurrency filter. @see RegisterFactory. + */ +REGISTER_FACTORY(AdaptiveConcurrencyFilterFactory, + Server::Configuration::NamedHttpFilterConfigFactory); + +} // namespace AdaptiveConcurrency +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/adaptive_concurrency/config.h b/source/extensions/filters/http/adaptive_concurrency/config.h new file mode 100644 index 000000000000..bce0e0fc6c71 --- /dev/null +++ b/source/extensions/filters/http/adaptive_concurrency/config.h @@ -0,0 +1,32 @@ +#pragma once + +#include "envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.pb.h" +#include "envoy/config/filter/http/adaptive_concurrency/v2alpha/adaptive_concurrency.pb.validate.h" + +#include "extensions/filters/http/common/factory_base.h" +#include "extensions/filters/http/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace AdaptiveConcurrency { + +/** + * Config registration for the adaptive concurrency limit filter. @see NamedHttpFilterConfigFactory. + */ +class AdaptiveConcurrencyFilterFactory + : public Common::FactoryBase< + envoy::config::filter::http::adaptive_concurrency::v2alpha::AdaptiveConcurrency> { +public: + AdaptiveConcurrencyFilterFactory() : FactoryBase(HttpFilterNames::get().AdaptiveConcurrency) {} + + Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::config::filter::http::adaptive_concurrency::v2alpha::AdaptiveConcurrency& + proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +} // namespace AdaptiveConcurrency +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/common/protobuf/utility_test.cc b/test/common/protobuf/utility_test.cc index 117b2cbc15c1..e0e3433f5f19 100644 --- a/test/common/protobuf/utility_test.cc +++ b/test/common/protobuf/utility_test.cc @@ -31,6 +31,14 @@ class ProtobufUtilityTest : public testing::Test { Api::ApiPtr api_; }; +TEST_F(ProtobufUtilityTest, convertPercentNaNDouble) { + envoy::api::v2::Cluster::CommonLbConfig common_config_; + common_config_.mutable_healthy_panic_threshold()->set_value( + std::numeric_limits::quiet_NaN()); + EXPECT_THROW(PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(common_config_, healthy_panic_threshold, 0.5), + EnvoyException); +} + TEST_F(ProtobufUtilityTest, convertPercentNaN) { envoy::api::v2::Cluster::CommonLbConfig common_config_; common_config_.mutable_healthy_panic_threshold()->set_value( @@ -38,8 +46,6 @@ TEST_F(ProtobufUtilityTest, convertPercentNaN) { EXPECT_THROW(PROTOBUF_PERCENT_TO_ROUNDED_INTEGER_OR_DEFAULT(common_config_, healthy_panic_threshold, 100, 50), EnvoyException); - EXPECT_THROW(PROTOBUF_PERCENT_TO_DOUBLE_OR_DEFAULT(common_config_, healthy_panic_threshold, 0.5), - EnvoyException); } namespace ProtobufPercentHelper { diff --git a/test/extensions/filters/http/adaptive_concurrency/BUILD b/test/extensions/filters/http/adaptive_concurrency/BUILD index 14aa0a108be2..870ff28d8766 100644 --- a/test/extensions/filters/http/adaptive_concurrency/BUILD +++ b/test/extensions/filters/http/adaptive_concurrency/BUILD @@ -26,3 +26,18 @@ envoy_extension_cc_test( "//test/test_common:utility_lib", ], ) + +envoy_extension_cc_test( + name = "adaptive_concurrency_integration_test", + srcs = [ + "adaptive_concurrency_filter_integration_test.cc", + "adaptive_concurrency_filter_integration_test.h", + ], + extension_name = "envoy.filters.http.adaptive_concurrency", + deps = [ + "//source/extensions/filters/http/adaptive_concurrency:config", + "//source/extensions/filters/http/fault:config", + "//test/integration:http_integration_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.cc b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.cc new file mode 100644 index 000000000000..ffd9089d20fc --- /dev/null +++ b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.cc @@ -0,0 +1,162 @@ +#include "test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.h" + +#include "common/http/header_map_impl.h" + +#include "test/integration/http_integration.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +void AdaptiveConcurrencyIntegrationTest::sendRequests(uint32_t request_count, + uint32_t num_forwarded) { + ASSERT_LE(num_forwarded, request_count); + + // TODO (tonya11en): + // We send header-only requests below because the adaptive concurrency filter will reject requests + // when decoding their headers. If we try to send data, there's no way to ensure that the filter + // doesn't respond between the client sending headers and data, invalidating the client's encoder + // stream. We should change this integration test to allow for the ability to test this scenario. + + // We expect these requests to reach the upstream. + for (uint32_t idx = 0; idx < num_forwarded; ++idx) { + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + responses_.push_back(std::move(response)); + upstream_connections_.emplace_back(); + upstream_requests_.emplace_back(); + + ASSERT_TRUE( + fake_upstreams_[0]->waitForHttpConnection(*dispatcher_, upstream_connections_.back())); + ASSERT_TRUE( + upstream_connections_.back()->waitForNewStream(*dispatcher_, upstream_requests_.back())); + + ASSERT_TRUE(upstream_requests_.back()->waitForEndStream(*dispatcher_)); + } + + // These requests should be blocked by the filter, so they never make it to the upstream. + auto blocked_counter = test_server_->counter(REQUEST_BLOCK_COUNTER_NAME)->value(); + for (uint32_t idx = 0; idx < request_count - num_forwarded; ++idx) { + auto response = codec_client_->makeHeaderOnlyRequest(default_request_headers_); + responses_.push_back(std::move(response)); + + test_server_->waitForCounterEq(REQUEST_BLOCK_COUNTER_NAME, ++blocked_counter); + + // These will remain nullptr. + upstream_connections_.emplace_back(); + upstream_requests_.emplace_back(); + } + + ASSERT_EQ(upstream_connections_.size(), upstream_requests_.size()); + ASSERT_EQ(responses_.size(), upstream_requests_.size()); +} + +void AdaptiveConcurrencyIntegrationTest::respondToAllRequests(uint32_t forwarded_count, + std::chrono::milliseconds latency) { + ASSERT_GE(responses_.size(), static_cast(forwarded_count)); + + timeSystem().sleep(latency); + + for (uint32_t idx = 0; idx < forwarded_count; ++idx) { + respondToRequest(true); + } + + while (!responses_.empty()) { + respondToRequest(false); + } +} + +void AdaptiveConcurrencyIntegrationTest::respondToRequest(bool expect_forwarded) { + ASSERT_EQ(upstream_connections_.size(), upstream_requests_.size()); + ASSERT_EQ(responses_.size(), upstream_requests_.size()); + + if (expect_forwarded) { + ASSERT_NE(upstream_connections_.front(), nullptr); + ASSERT_NE(upstream_requests_.front(), nullptr); + ASSERT_TRUE(upstream_requests_.front()->waitForEndStream(*dispatcher_)); + upstream_requests_.front()->encodeHeaders(default_response_headers_, false); + upstream_requests_.front()->encodeData(1, true); + } + + responses_.front()->waitForEndStream(); + + if (expect_forwarded) { + EXPECT_TRUE(upstream_requests_.front()->complete()); + } + + EXPECT_TRUE(responses_.front()->complete()); + + if (expect_forwarded) { + verifyResponseForwarded(std::move(responses_.front())); + ASSERT_TRUE(upstream_connections_.front()->close()); + ASSERT_TRUE(upstream_connections_.front()->waitForDisconnect()); + } else { + verifyResponseBlocked(std::move(responses_.front())); + } + + upstream_connections_.pop_front(); + upstream_requests_.pop_front(); + responses_.pop_front(); +} + +INSTANTIATE_TEST_SUITE_P(IpVersions, AdaptiveConcurrencyIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest())); + +/** + * Test a single request returns successfully. + */ +TEST_P(AdaptiveConcurrencyIntegrationTest, TestConcurrency1) { + customInit(); + + EXPECT_EQ(0, test_server_->counter(REQUEST_BLOCK_COUNTER_NAME)->value()); + sendRequests(2, 1); + respondToAllRequests(1, std::chrono::milliseconds(5)); + test_server_->waitForCounterEq(REQUEST_BLOCK_COUNTER_NAME, 1); +} + +/** + * Test many requests, where only a single request returns 200 during the minRTT window. + */ +TEST_P(AdaptiveConcurrencyIntegrationTest, TestManyConcurrency1) { + customInit(); + + EXPECT_EQ(0, test_server_->counter(REQUEST_BLOCK_COUNTER_NAME)->value()); + sendRequests(10, 1); + respondToAllRequests(1, std::chrono::milliseconds(5)); + test_server_->waitForCounterEq(REQUEST_BLOCK_COUNTER_NAME, 9); +} + +/** + * TODO: Test the ability to increase/decrease the concurrency limit with request latencies based on + * the minRTT value. + * + * See PR #8405. + * + * Previous attempts at this test took a long time when using simulated time, which resulted in + * intermittent timeouts in CI. + */ + +/** + * TODO: Test the ability to enforce the concurrency limit outside of the minRTT calculation window. + * + * See PR #8405. + * + * Previous attempts at this test would hang during waitForHttpConnection after successfully sending + * several requests to inflate the minRTT value. Alternative approaches that circumvented the need + * for manually waiting included: + * + * - Using a fault filter to inject delay into requests after passing the adaptive concurrency + * filter. This fails when using simulated time due to the fault filter's delay mechanism not + * being governed by the simulated time class. This required usage of real time, which sacrificed + * determinism. + * + * - Buffering requests at the fake upstream and releasing them manually. Buffering via simulated + * time and releasing by advancing time does not work due to the only_one_thread.h assertions + * requiring simulated time to advance on a single thread. Buffering via a request queue and + * changes to the fake upstream requires too many changes to the fake upstream to be worth the + * investment of time, since it would be more worthwhile to overhaul the integration test + * framework to be event-driven rather than waitFor* driven. + */ + +} // namespace Envoy diff --git a/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.h b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.h new file mode 100644 index 000000000000..d7358d65df1b --- /dev/null +++ b/test/extensions/filters/http/adaptive_concurrency/adaptive_concurrency_filter_integration_test.h @@ -0,0 +1,80 @@ +#include "test/integration/http_integration.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" + +#include "gtest/gtest.h" + +namespace Envoy { + +const std::string ADAPTIVE_CONCURRENCY_CONFIG = + R"EOF( +name: envoy.filters.http.adaptive_concurrency +config: + gradient_controller_config: + sample_aggregate_percentile: + value: 50 + concurrency_limit_params: + concurrency_update_interval: 0.1s + min_rtt_calc_params: + interval: 30s + request_count: 50 +)EOF"; + +const std::string CONCURRENCY_LIMIT_GAUGE_NAME = + "http.config_test.adaptive_concurrency.gradient_controller.concurrency_limit"; +const std::string REQUEST_BLOCK_COUNTER_NAME = + "http.config_test.adaptive_concurrency.gradient_controller.rq_blocked"; +const std::string MIN_RTT_GAUGE_NAME = + "http.config_test.adaptive_concurrency.gradient_controller.min_rtt_msecs"; + +class AdaptiveConcurrencyIntegrationTest + : public testing::TestWithParam, + public Event::TestUsingSimulatedTime, + public HttpIntegrationTest { +public: + AdaptiveConcurrencyIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam()) {} + + void customInit() { + setDownstreamProtocol(Http::CodecClient::Type::HTTP2); + setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); + config_helper_.addFilter(ADAPTIVE_CONCURRENCY_CONFIG); + initialize(); + codec_client_ = makeHttpConnection(lookupPort("http")); + } + + void TearDown() override { + HttpIntegrationTest::cleanupUpstreamAndDownstream(); + codec_client_->close(); + codec_client_.reset(); + } + +protected: + // Send some number of requests with 'delay_ms' specifying the amount of time the fault filter + // will delay them. + void sendRequests(uint32_t request_count, uint32_t num_forwarded); + + // Waits for a specified duration and then responds to all queued up requests in a FIFO manner. + // Asserts that the expected number of requests are forwarded through the filter. The oldest + // requests are the forwarded requests. + // + // Note: For interleaved forwarded/blocked requests, use respondToRequest() directly. + void respondToAllRequests(uint32_t forwarded_count, std::chrono::milliseconds latency); + + // Responds to a single request in a FIFO manner. Asserts the forwarding expectation. + void respondToRequest(bool expect_forwarded); + + void verifyResponseForwarded(IntegrationStreamDecoderPtr response) { + EXPECT_EQ("200", response->headers().Status()->value().getStringView()); + } + + void verifyResponseBlocked(IntegrationStreamDecoderPtr response) { + EXPECT_EQ("503", response->headers().Status()->value().getStringView()); + } + + std::deque responses_; + std::deque upstream_requests_; + std::deque upstream_connections_; +}; + +} // namespace Envoy diff --git a/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc b/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc index f3de2716299b..e8beb11b7862 100644 --- a/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc +++ b/test/extensions/filters/http/adaptive_concurrency/concurrency_controller/gradient_controller_test.cc @@ -47,12 +47,12 @@ class GradientControllerTest : public testing::Test { } protected: - GradientControllerConfigSharedPtr makeConfig(const std::string& yaml_config) { + GradientControllerConfig makeConfig(const std::string& yaml_config) { envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig proto = TestUtility::parseYaml< envoy::config::filter::http::adaptive_concurrency::v2alpha::GradientControllerConfig>( yaml_config); - return std::make_shared(proto); + return proto; } // Helper function that will attempt to pull forwarding decisions. @@ -68,7 +68,7 @@ class GradientControllerTest : public testing::Test { const std::string& yaml_config, std::chrono::milliseconds latency = std::chrono::milliseconds(5)) { const auto config = makeConfig(yaml_config); - for (uint32_t i = 0; i <= config->minRTTAggregateRequestCount(); ++i) { + for (uint32_t i = 0; i <= config.minRTTAggregateRequestCount(); ++i) { tryForward(controller, true); controller->recordLatencySample(latency); } From 2d2c9a5fe1614715a98ba9d93a057c3f37c33590 Mon Sep 17 00:00:00 2001 From: yanavlasov Date: Tue, 1 Oct 2019 13:37:34 -0400 Subject: [PATCH 26/32] Replace deprecated thread annotations macros (#8437) Clean-up stragglers that were not included in the original commit. Abseil thread annotation macros are now prefixed by ABSL_. There is no semantic change; this is just a rename. Signed-off-by: Yan Avlasov --- source/common/common/perf_annotation.h | 2 +- source/common/grpc/context_impl.h | 4 ++-- source/common/grpc/google_async_client_impl.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/common/common/perf_annotation.h b/source/common/common/perf_annotation.h index d2f06ebc28ac..e21b012d0a75 100644 --- a/source/common/common/perf_annotation.h +++ b/source/common/common/perf_annotation.h @@ -136,7 +136,7 @@ class PerfAnnotationContext { // Maps {category, description} to DurationStats. #if PERF_THREAD_SAFE - DurationStatsMap duration_stats_map_ GUARDED_BY(mutex_); + DurationStatsMap duration_stats_map_ ABSL_GUARDED_BY(mutex_); Thread::MutexBasicLockable mutex_; #else DurationStatsMap duration_stats_map_; diff --git a/source/common/grpc/context_impl.h b/source/common/grpc/context_impl.h index accf3d9c89fe..9a8a716fcbea 100644 --- a/source/common/grpc/context_impl.h +++ b/source/common/grpc/context_impl.h @@ -56,8 +56,8 @@ class ContextImpl : public Context { Stats::SymbolTable& symbol_table_; mutable Thread::MutexBasicLockable mutex_; - Stats::StatNamePool stat_name_pool_ GUARDED_BY(mutex_); - StringMap stat_name_map_ GUARDED_BY(mutex_); + Stats::StatNamePool stat_name_pool_ ABSL_GUARDED_BY(mutex_); + StringMap stat_name_map_ ABSL_GUARDED_BY(mutex_); const Stats::StatName grpc_; const Stats::StatName grpc_web_; const Stats::StatName success_; diff --git a/source/common/grpc/google_async_client_impl.h b/source/common/grpc/google_async_client_impl.h index 00abc1d5a8f3..272747382cdf 100644 --- a/source/common/grpc/google_async_client_impl.h +++ b/source/common/grpc/google_async_client_impl.h @@ -295,7 +295,7 @@ class GoogleAsyncStreamImpl : public RawAsyncStream, // Queue of completed (op, ok) passed from completionThread() to // handleOpCompletion(). std::deque> - completed_ops_ GUARDED_BY(completed_ops_lock_); + completed_ops_ ABSL_GUARDED_BY(completed_ops_lock_); Thread::MutexBasicLockable completed_ops_lock_; friend class GoogleAsyncClientImpl; From a662ad5eebc8f6b954169fd33d124d8c8c2301ed Mon Sep 17 00:00:00 2001 From: Lizan Zhou Date: Tue, 1 Oct 2019 14:13:05 -0700 Subject: [PATCH 27/32] refactor: move TransportSocketOptions into LoadBalancerContext (#8424) Description: Needed for #2690 to use TransportSocketOptions in HTTP cluster. Risk Level: Low (refactor only) Testing: existing Docs Changes: N/A Release Notes: N/A Signed-off-by: Lizan Zhou --- include/envoy/upstream/cluster_manager.h | 12 +- include/envoy/upstream/load_balancer.h | 6 + source/common/tcp_proxy/tcp_proxy.cc | 6 +- source/common/tcp_proxy/tcp_proxy.h | 5 + .../common/upstream/cluster_manager_impl.cc | 31 +-- source/common/upstream/cluster_manager_impl.h | 17 +- source/common/upstream/load_balancer_impl.h | 6 +- .../network/dubbo_proxy/router/router_impl.cc | 2 +- .../thrift_proxy/router/router_impl.cc | 2 +- .../stat_sinks/common/statsd/statsd.cc | 2 +- .../config_validation/cluster_manager.cc | 5 +- .../config_validation/cluster_manager.h | 3 +- .../network/filter_manager_impl_test.cc | 2 +- test/common/tcp_proxy/tcp_proxy_test.cc | 32 +-- .../upstream/cluster_manager_impl_test.cc | 198 ++++++++---------- .../network/dubbo_proxy/router_test.cc | 2 +- .../network/thrift_proxy/router_test.cc | 2 +- .../tcp_conn_pool_integration_test.cc | 2 +- test/mocks/upstream/load_balancer_context.h | 1 + test/mocks/upstream/mocks.cc | 2 +- test/mocks/upstream/mocks.h | 11 +- .../config_validation/cluster_manager_test.cc | 2 +- 22 files changed, 165 insertions(+), 186 deletions(-) diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 7dfde10cc03c..d911f7982137 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -135,10 +135,9 @@ class ClusterManager { * Can return nullptr if there is no host available in the cluster or if the cluster does not * exist. */ - virtual Tcp::ConnectionPool::Instance* - tcpConnPoolForCluster(const std::string& cluster, ResourcePriority priority, - LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr transport_socket_options) PURE; + virtual Tcp::ConnectionPool::Instance* tcpConnPoolForCluster(const std::string& cluster, + ResourcePriority priority, + LoadBalancerContext* context) PURE; /** * Allocate a load balanced TCP connection for a cluster. The created connection is already @@ -148,9 +147,8 @@ class ClusterManager { * Returns both a connection and the host that backs the connection. Both can be nullptr if there * is no host available in the cluster. */ - virtual Host::CreateConnectionData - tcpConnForCluster(const std::string& cluster, LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr transport_socket_options) PURE; + virtual Host::CreateConnectionData tcpConnForCluster(const std::string& cluster, + LoadBalancerContext* context) PURE; /** * Returns a client that can be used to make async HTTP calls against the given cluster. The diff --git a/include/envoy/upstream/load_balancer.h b/include/envoy/upstream/load_balancer.h index c631997b4323..5d815818c1f4 100644 --- a/include/envoy/upstream/load_balancer.h +++ b/include/envoy/upstream/load_balancer.h @@ -4,6 +4,7 @@ #include #include "envoy/common/pure.h" +#include "envoy/network/transport_socket.h" #include "envoy/router/router.h" #include "envoy/upstream/types.h" #include "envoy/upstream/upstream.h" @@ -76,6 +77,11 @@ class LoadBalancerContext { * Returns the set of socket options which should be applied on upstream connections */ virtual Network::Socket::OptionsSharedPtr upstreamSocketOptions() const PURE; + + /** + * Returns the transport socket options which should be applied on upstream connections + */ + virtual Network::TransportSocketOptionsSharedPtr upstreamTransportSocketOptions() const PURE; }; /** diff --git a/source/common/tcp_proxy/tcp_proxy.cc b/source/common/tcp_proxy/tcp_proxy.cc index aee0dbb70fb5..0a1987fe7a13 100644 --- a/source/common/tcp_proxy/tcp_proxy.cc +++ b/source/common/tcp_proxy/tcp_proxy.cc @@ -368,20 +368,18 @@ Network::FilterStatus Filter::initializeUpstreamConnection() { return Network::FilterStatus::StopIteration; } - Network::TransportSocketOptionsSharedPtr transport_socket_options; - if (downstreamConnection() && downstreamConnection()->streamInfo().filterState().hasData( UpstreamServerName::key())) { const auto& original_requested_server_name = downstreamConnection()->streamInfo().filterState().getDataReadOnly( UpstreamServerName::key()); - transport_socket_options = std::make_shared( + transport_socket_options_ = std::make_shared( original_requested_server_name.value()); } Tcp::ConnectionPool::Instance* conn_pool = cluster_manager_.tcpConnPoolForCluster( - cluster_name, Upstream::ResourcePriority::Default, this, transport_socket_options); + cluster_name, Upstream::ResourcePriority::Default, this); if (!conn_pool) { // Either cluster is unknown or there are no healthy hosts. tcpConnPoolForCluster() increments // cluster->stats().upstream_cx_none_healthy in the latter case. diff --git a/source/common/tcp_proxy/tcp_proxy.h b/source/common/tcp_proxy/tcp_proxy.h index 928accb654ef..a39cb0f2dcc8 100644 --- a/source/common/tcp_proxy/tcp_proxy.h +++ b/source/common/tcp_proxy/tcp_proxy.h @@ -200,6 +200,10 @@ class Filter : public Network::ReadFilter, return &read_callbacks_->connection(); } + Network::TransportSocketOptionsSharedPtr upstreamTransportSocketOptions() const override { + return transport_socket_options_; + } + // These two functions allow enabling/disabling reads on the upstream and downstream connections. // They are called by the Downstream/Upstream Watermark callbacks to limit buffering. void readDisableUpstream(bool disable); @@ -282,6 +286,7 @@ class Filter : public Network::ReadFilter, std::shared_ptr upstream_callbacks_; // shared_ptr required for passing as a // read filter. StreamInfo::StreamInfoImpl stream_info_; + Network::TransportSocketOptionsSharedPtr transport_socket_options_; uint32_t connect_attempts_{}; bool connecting_{}; }; diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index c873db74fcf5..70b4c9eb9c01 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -739,9 +739,9 @@ ClusterManagerImpl::httpConnPoolForCluster(const std::string& cluster, ResourceP return entry->second->connPool(priority, protocol, context); } -Tcp::ConnectionPool::Instance* ClusterManagerImpl::tcpConnPoolForCluster( - const std::string& cluster, ResourcePriority priority, LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr transport_socket_options) { +Tcp::ConnectionPool::Instance* +ClusterManagerImpl::tcpConnPoolForCluster(const std::string& cluster, ResourcePriority priority, + LoadBalancerContext* context) { ThreadLocalClusterManagerImpl& cluster_manager = tls_->getTyped(); auto entry = cluster_manager.thread_local_clusters_.find(cluster); @@ -750,7 +750,7 @@ Tcp::ConnectionPool::Instance* ClusterManagerImpl::tcpConnPoolForCluster( } // Select a host and create a connection pool for it if it does not already exist. - return entry->second->tcpConnPool(priority, context, transport_socket_options); + return entry->second->tcpConnPool(priority, context); } void ClusterManagerImpl::postThreadLocalDrainConnections(const Cluster& cluster, @@ -780,9 +780,8 @@ void ClusterManagerImpl::postThreadLocalHealthFailure(const HostSharedPtr& host) [this, host] { ThreadLocalClusterManagerImpl::onHostHealthFailure(host, *tls_); }); } -Host::CreateConnectionData ClusterManagerImpl::tcpConnForCluster( - const std::string& cluster, LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr transport_socket_options) { +Host::CreateConnectionData ClusterManagerImpl::tcpConnForCluster(const std::string& cluster, + LoadBalancerContext* context) { ThreadLocalClusterManagerImpl& cluster_manager = tls_->getTyped(); auto entry = cluster_manager.thread_local_clusters_.find(cluster); @@ -792,8 +791,9 @@ Host::CreateConnectionData ClusterManagerImpl::tcpConnForCluster( HostConstSharedPtr logical_host = entry->second->lb_->chooseHost(context); if (logical_host) { - auto conn_info = logical_host->createConnection(cluster_manager.thread_local_dispatcher_, - nullptr, transport_socket_options); + auto conn_info = logical_host->createConnection( + cluster_manager.thread_local_dispatcher_, nullptr, + context == nullptr ? nullptr : context->upstreamTransportSocketOptions()); if ((entry->second->cluster_info_->features() & ClusterInfo::Features::CLOSE_CONNECTIONS_ON_HOST_HEALTH_FAILURE) && conn_info.connection_ != nullptr) { @@ -1227,8 +1227,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::connPool( Tcp::ConnectionPool::Instance* ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::tcpConnPool( - ResourcePriority priority, LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr transport_socket_options) { + ResourcePriority priority, LoadBalancerContext* context) { HostConstSharedPtr host = lb_->chooseHost(context); if (!host) { ENVOY_LOG(debug, "no healthy host for TCP connection pool"); @@ -1243,7 +1242,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::tcpConnPool( // This allows socket options to control connection pooling so that connections with // different options are not pooled together. bool have_options = false; - if (context && context->downstreamConnection()) { + if (context != nullptr && context->downstreamConnection()) { const Network::ConnectionSocket::OptionsSharedPtr& options = context->downstreamConnection()->socketOptions(); if (options) { @@ -1254,8 +1253,10 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::tcpConnPool( } } - if (transport_socket_options != nullptr) { - transport_socket_options->hashKey(hash_key); + bool have_transport_socket_options = false; + if (context != nullptr && context->upstreamTransportSocketOptions() != nullptr) { + have_transport_socket_options = true; + context->upstreamTransportSocketOptions()->hashKey(hash_key); } TcpConnPoolsContainer& container = parent_.host_tcp_conn_pool_map_[host]; @@ -1263,7 +1264,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::tcpConnPool( container.pools_[hash_key] = parent_.parent_.factory_.allocateTcpConnPool( parent_.thread_local_dispatcher_, host, priority, have_options ? context->downstreamConnection()->socketOptions() : nullptr, - transport_socket_options); + have_transport_socket_options ? context->upstreamTransportSocketOptions() : nullptr); } return container.pools_[hash_key].get(); diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index cb45bb14aca7..4838178804d6 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -206,13 +206,11 @@ class ClusterManagerImpl : public ClusterManager, Logger::LoggableclusterName(), Upstream::ResourcePriority::Default, this, nullptr); + route_entry_->clusterName(), Upstream::ResourcePriority::Default, this); if (!conn_pool) { callbacks_->sendLocalReply( AppException( diff --git a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc index 81a9b015ee65..afff31455451 100644 --- a/source/extensions/filters/network/thrift_proxy/router/router_impl.cc +++ b/source/extensions/filters/network/thrift_proxy/router/router_impl.cc @@ -246,7 +246,7 @@ FilterStatus Router::messageBegin(MessageMetadataSharedPtr metadata) { ASSERT(protocol != ProtocolType::Auto); Tcp::ConnectionPool::Instance* conn_pool = cluster_manager_.tcpConnPoolForCluster( - route_entry_->clusterName(), Upstream::ResourcePriority::Default, this, nullptr); + route_entry_->clusterName(), Upstream::ResourcePriority::Default, this); if (!conn_pool) { callbacks_->sendLocalReply( AppException(AppExceptionType::InternalError, diff --git a/source/extensions/stat_sinks/common/statsd/statsd.cc b/source/extensions/stat_sinks/common/statsd/statsd.cc index 4dc3a5dae872..8ffcbe05f898 100644 --- a/source/extensions/stat_sinks/common/statsd/statsd.cc +++ b/source/extensions/stat_sinks/common/statsd/statsd.cc @@ -235,7 +235,7 @@ void TcpStatsdSink::TlsSink::write(Buffer::Instance& buffer) { if (!connection_) { Upstream::Host::CreateConnectionData info = - parent_.cluster_manager_.tcpConnForCluster(parent_.cluster_info_->name(), nullptr, nullptr); + parent_.cluster_manager_.tcpConnForCluster(parent_.cluster_info_->name(), nullptr); if (!info.connection_) { buffer.drain(buffer.length()); return; diff --git a/source/server/config_validation/cluster_manager.cc b/source/server/config_validation/cluster_manager.cc index c5fdc16db276..cf20a6c0e221 100644 --- a/source/server/config_validation/cluster_manager.cc +++ b/source/server/config_validation/cluster_manager.cc @@ -38,9 +38,8 @@ ValidationClusterManager::httpConnPoolForCluster(const std::string&, ResourcePri return nullptr; } -Host::CreateConnectionData -ValidationClusterManager::tcpConnForCluster(const std::string&, LoadBalancerContext*, - Network::TransportSocketOptionsSharedPtr) { +Host::CreateConnectionData ValidationClusterManager::tcpConnForCluster(const std::string&, + LoadBalancerContext*) { return Host::CreateConnectionData{nullptr, nullptr}; } diff --git a/source/server/config_validation/cluster_manager.h b/source/server/config_validation/cluster_manager.h index 5a7f29955475..0fe33294d190 100644 --- a/source/server/config_validation/cluster_manager.h +++ b/source/server/config_validation/cluster_manager.h @@ -62,8 +62,7 @@ class ValidationClusterManager : public ClusterManagerImpl { Http::ConnectionPool::Instance* httpConnPoolForCluster(const std::string&, ResourcePriority, Http::Protocol, LoadBalancerContext*) override; - Host::CreateConnectionData tcpConnForCluster(const std::string&, LoadBalancerContext*, - Network::TransportSocketOptionsSharedPtr) override; + Host::CreateConnectionData tcpConnForCluster(const std::string&, LoadBalancerContext*) override; Http::AsyncClient& httpAsyncClientForCluster(const std::string&) override; private: diff --git a/test/common/network/filter_manager_impl_test.cc b/test/common/network/filter_manager_impl_test.cc index f713a79a63e3..d03518f363b2 100644 --- a/test/common/network/filter_manager_impl_test.cc +++ b/test/common/network/filter_manager_impl_test.cc @@ -409,7 +409,7 @@ stat_prefix: name EXPECT_EQ(manager.initializeReadFilters(), true); - EXPECT_CALL(factory_context.cluster_manager_, tcpConnPoolForCluster("fake_cluster", _, _, _)) + EXPECT_CALL(factory_context.cluster_manager_, tcpConnPoolForCluster("fake_cluster", _, _)) .WillOnce(Return(&conn_pool)); request_callbacks->complete(Extensions::Filters::Common::RateLimit::LimitStatus::OK, nullptr); diff --git a/test/common/tcp_proxy/tcp_proxy_test.cc b/test/common/tcp_proxy/tcp_proxy_test.cc index 07c505c2e901..85af207ede4f 100644 --- a/test/common/tcp_proxy/tcp_proxy_test.cc +++ b/test/common/tcp_proxy/tcp_proxy_test.cc @@ -421,8 +421,7 @@ class TcpProxyTest : public testing::Test { { testing::InSequence sequence; for (uint32_t i = 0; i < connections; i++) { - EXPECT_CALL(factory_context_.cluster_manager_, - tcpConnPoolForCluster("fake_cluster", _, _, _)) + EXPECT_CALL(factory_context_.cluster_manager_, tcpConnPoolForCluster("fake_cluster", _, _)) .WillOnce(Return(&conn_pool_)) .RetiresOnSaturation(); EXPECT_CALL(conn_pool_, newConnection(_)) @@ -433,7 +432,7 @@ class TcpProxyTest : public testing::Test { })) .RetiresOnSaturation(); } - EXPECT_CALL(factory_context_.cluster_manager_, tcpConnPoolForCluster("fake_cluster", _, _, _)) + EXPECT_CALL(factory_context_.cluster_manager_, tcpConnPoolForCluster("fake_cluster", _, _)) .WillRepeatedly(Return(nullptr)); } @@ -1199,7 +1198,7 @@ TEST_F(TcpProxyRoutingTest, RoutableConnection) { connection_.local_address_ = std::make_shared("1.2.3.4", 9999); // Expect filter to try to open a connection to specified cluster. - EXPECT_CALL(factory_context_.cluster_manager_, tcpConnPoolForCluster("fake_cluster", _, _, _)) + EXPECT_CALL(factory_context_.cluster_manager_, tcpConnPoolForCluster("fake_cluster", _, _)) .WillOnce(Return(nullptr)); filter_->onNewConnection(); @@ -1221,7 +1220,7 @@ TEST_F(TcpProxyRoutingTest, UseClusterFromPerConnectionCluster) { // Expect filter to try to open a connection to specified cluster. EXPECT_CALL(factory_context_.cluster_manager_, - tcpConnPoolForCluster("filter_state_cluster", _, _, _)) + tcpConnPoolForCluster("filter_state_cluster", _, _)) .WillOnce(Return(nullptr)); filter_->onNewConnection(); @@ -1241,17 +1240,18 @@ TEST_F(TcpProxyRoutingTest, UpstreamServerName) { // Expect filter to try to open a connection to a cluster with the transport socket options with // override-server-name - EXPECT_CALL(factory_context_.cluster_manager_, tcpConnPoolForCluster(_, _, _, _)) - .WillOnce(Invoke([](const std::string& cluster, Upstream::ResourcePriority, - Upstream::LoadBalancerContext*, - Network::TransportSocketOptionsSharedPtr transport_socket_options) - -> Tcp::ConnectionPool::Instance* { - EXPECT_EQ(cluster, "fake_cluster"); - EXPECT_NE(transport_socket_options, nullptr); - EXPECT_TRUE(transport_socket_options->serverNameOverride().has_value()); - EXPECT_EQ(transport_socket_options->serverNameOverride().value(), "www.example.com"); - return nullptr; - })); + EXPECT_CALL(factory_context_.cluster_manager_, tcpConnPoolForCluster(_, _, _)) + .WillOnce( + Invoke([](const std::string& cluster, Upstream::ResourcePriority, + Upstream::LoadBalancerContext* context) -> Tcp::ConnectionPool::Instance* { + EXPECT_EQ(cluster, "fake_cluster"); + Network::TransportSocketOptionsSharedPtr transport_socket_options = + context->upstreamTransportSocketOptions(); + EXPECT_NE(transport_socket_options, nullptr); + EXPECT_TRUE(transport_socket_options->serverNameOverride().has_value()); + EXPECT_EQ(transport_socket_options->serverNameOverride().value(), "www.example.com"); + return nullptr; + })); // Port 9999 is within the specified destination port range. connection_.local_address_ = std::make_shared("1.2.3.4", 9999); diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 1534ba3f3b2d..3aed95db6298 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -963,17 +963,16 @@ TEST_F(ClusterManagerImplTest, UnknownCluster) { EXPECT_EQ(nullptr, cluster_manager_->get("hello")); EXPECT_EQ(nullptr, cluster_manager_->httpConnPoolForCluster("hello", ResourcePriority::Default, Http::Protocol::Http2, nullptr)); - Network::TransportSocketOptionsSharedPtr transport_socket_options; - EXPECT_EQ(nullptr, cluster_manager_->tcpConnPoolForCluster("hello", ResourcePriority::Default, - nullptr, transport_socket_options)); - EXPECT_THROW(cluster_manager_->tcpConnForCluster("hello", nullptr, transport_socket_options), - EnvoyException); + EXPECT_EQ(nullptr, + cluster_manager_->tcpConnPoolForCluster("hello", ResourcePriority::Default, nullptr)); + EXPECT_THROW(cluster_manager_->tcpConnForCluster("hello", nullptr), EnvoyException); - transport_socket_options = std::make_shared("example.com"); + NiceMock example_com_context; + ON_CALL(example_com_context, upstreamTransportSocketOptions()) + .WillByDefault(Return(std::make_shared("example.com"))); EXPECT_EQ(nullptr, cluster_manager_->tcpConnPoolForCluster("hello", ResourcePriority::Default, - nullptr, transport_socket_options)); - EXPECT_THROW(cluster_manager_->tcpConnForCluster("hello", nullptr, transport_socket_options), - EnvoyException); + &example_com_context)); + EXPECT_THROW(cluster_manager_->tcpConnForCluster("hello", &example_com_context), EnvoyException); EXPECT_THROW(cluster_manager_->httpAsyncClientForCluster("hello"), EnvoyException); factory_.tls_.shutdownThread(); @@ -1006,7 +1005,7 @@ TEST_F(ClusterManagerImplTest, VerifyBufferLimits) { EXPECT_CALL(*connection, setBufferLimits(8192)); EXPECT_CALL(factory_.tls_.dispatcher_, createClientConnection_(_, _, _, _)) .WillOnce(Return(connection)); - auto conn_data = cluster_manager_->tcpConnForCluster("cluster_1", nullptr, nullptr); + auto conn_data = cluster_manager_->tcpConnForCluster("cluster_1", nullptr); EXPECT_EQ(connection, conn_data.connection_.get()); factory_.tls_.shutdownThread(); } @@ -1469,7 +1468,7 @@ TEST_F(ClusterManagerImplTest, DynamicAddRemove) { Tcp::ConnectionPool::MockInstance* cp2 = new Tcp::ConnectionPool::MockInstance(); EXPECT_CALL(factory_, allocateTcpConnPool_(_)).WillOnce(Return(cp2)); EXPECT_EQ(cp2, cluster_manager_->tcpConnPoolForCluster("fake_cluster", ResourcePriority::Default, - nullptr, nullptr)); + nullptr)); Network::MockClientConnection* connection = new Network::MockClientConnection(); ON_CALL(*cluster2->info_, features()) @@ -1478,7 +1477,7 @@ TEST_F(ClusterManagerImplTest, DynamicAddRemove) { .WillOnce(Return(connection)); EXPECT_CALL(*connection, setBufferLimits(_)); EXPECT_CALL(*connection, addConnectionCallbacks(_)); - auto conn_info = cluster_manager_->tcpConnForCluster("fake_cluster", nullptr, nullptr); + auto conn_info = cluster_manager_->tcpConnForCluster("fake_cluster", nullptr); EXPECT_EQ(conn_info.connection_.get(), connection); // Now remove the cluster. This should drain the connection pools, but not affect @@ -1683,8 +1682,7 @@ TEST_F(ClusterManagerImplTest, CloseTcpConnectionPoolsOnHealthFailure) { create(parseBootstrapFromV2Json(json)); EXPECT_CALL(factory_, allocateTcpConnPool_(_)).WillOnce(Return(cp1)); - cluster_manager_->tcpConnPoolForCluster("some_cluster", ResourcePriority::Default, nullptr, - nullptr); + cluster_manager_->tcpConnPoolForCluster("some_cluster", ResourcePriority::Default, nullptr); outlier_detector.runCallbacks(test_host); health_checker.runCallbacks(test_host, HealthTransition::Unchanged); @@ -1694,8 +1692,7 @@ TEST_F(ClusterManagerImplTest, CloseTcpConnectionPoolsOnHealthFailure) { outlier_detector.runCallbacks(test_host); EXPECT_CALL(factory_, allocateTcpConnPool_(_)).WillOnce(Return(cp2)); - cluster_manager_->tcpConnPoolForCluster("some_cluster", ResourcePriority::High, nullptr, - nullptr); + cluster_manager_->tcpConnPoolForCluster("some_cluster", ResourcePriority::High, nullptr); } // Order of these calls is implementation dependent, so can't sequence them! @@ -1757,7 +1754,7 @@ TEST_F(ClusterManagerImplTest, CloseTcpConnectionsOnHealthFailure) { EXPECT_CALL(factory_.tls_.dispatcher_, createClientConnection_(_, _, _, _)) .WillOnce(Return(connection1)); - conn_info1 = cluster_manager_->tcpConnForCluster("some_cluster", nullptr, nullptr); + conn_info1 = cluster_manager_->tcpConnForCluster("some_cluster", nullptr); outlier_detector.runCallbacks(test_host); health_checker.runCallbacks(test_host, HealthTransition::Unchanged); @@ -1769,11 +1766,11 @@ TEST_F(ClusterManagerImplTest, CloseTcpConnectionsOnHealthFailure) { connection1 = new NiceMock(); EXPECT_CALL(factory_.tls_.dispatcher_, createClientConnection_(_, _, _, _)) .WillOnce(Return(connection1)); - conn_info1 = cluster_manager_->tcpConnForCluster("some_cluster", nullptr, nullptr); + conn_info1 = cluster_manager_->tcpConnForCluster("some_cluster", nullptr); EXPECT_CALL(factory_.tls_.dispatcher_, createClientConnection_(_, _, _, _)) .WillOnce(Return(connection2)); - conn_info2 = cluster_manager_->tcpConnForCluster("some_cluster", nullptr, nullptr); + conn_info2 = cluster_manager_->tcpConnForCluster("some_cluster", nullptr); } // Order of these calls is implementation dependent, so can't sequence them! @@ -1830,7 +1827,7 @@ TEST_F(ClusterManagerImplTest, DoNotCloseTcpConnectionsOnHealthFailure) { EXPECT_CALL(factory_.tls_.dispatcher_, createClientConnection_(_, _, _, _)) .WillOnce(Return(connection1)); - conn_info1 = cluster_manager_->tcpConnForCluster("some_cluster", nullptr, nullptr); + conn_info1 = cluster_manager_->tcpConnForCluster("some_cluster", nullptr); outlier_detector.runCallbacks(test_host); health_checker.runCallbacks(test_host, HealthTransition::Unchanged); @@ -1879,9 +1876,8 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemove) { EXPECT_EQ(nullptr, cluster_manager_->httpConnPoolForCluster( "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); EXPECT_EQ(nullptr, cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, - nullptr, nullptr)); - EXPECT_EQ(nullptr, - cluster_manager_->tcpConnForCluster("cluster_1", nullptr, nullptr).connection_); + nullptr)); + EXPECT_EQ(nullptr, cluster_manager_->tcpConnForCluster("cluster_1", nullptr).connection_); EXPECT_EQ(3UL, factory_.stats_.counter("cluster.cluster_1.upstream_cx_none_healthy").value()); // Set up for an initialize callback. @@ -1928,18 +1924,14 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemove) { .WillRepeatedly(ReturnNew()); // This should provide us a CP for each of the above hosts. - Tcp::ConnectionPool::MockInstance* tcp1 = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp2 = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp1_high = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::High, nullptr, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp2_high = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::High, nullptr, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp1 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp2 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp1_high = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::High, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp2_high = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::High, nullptr)); EXPECT_NE(tcp1, tcp2); EXPECT_NE(tcp1_high, tcp2_high); @@ -1973,12 +1965,10 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemove) { EXPECT_EQ(cp2, cp3); EXPECT_EQ(cp2_high, cp3_high); - Tcp::ConnectionPool::MockInstance* tcp3 = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp3_high = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::High, nullptr, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp3 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp3_high = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::High, nullptr)); EXPECT_EQ(tcp2, tcp3); EXPECT_EQ(tcp2_high, tcp3_high); @@ -2022,33 +2012,30 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemoveWithTls) { create(parseBootstrapFromV2Yaml(yaml)); EXPECT_FALSE(cluster_manager_->get("cluster_1")->info()->addedViaApi()); - Network::TransportSocketOptionsSharedPtr transport_socket_options_example_com( - new Network::TransportSocketOptionsImpl("example.com")); - Network::TransportSocketOptionsSharedPtr transport_socket_options_ibm_com( - new Network::TransportSocketOptionsImpl("ibm.com")); + NiceMock example_com_context; + ON_CALL(example_com_context, upstreamTransportSocketOptions()) + .WillByDefault(Return(std::make_shared("example.com"))); + + NiceMock ibm_com_context; + ON_CALL(ibm_com_context, upstreamTransportSocketOptions()) + .WillByDefault(Return(std::make_shared("ibm.com"))); // Test for no hosts returning the correct values before we have hosts. EXPECT_EQ(nullptr, cluster_manager_->httpConnPoolForCluster( "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); EXPECT_EQ(nullptr, cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, - nullptr, nullptr)); - EXPECT_EQ(nullptr, - cluster_manager_->tcpConnForCluster("cluster_1", nullptr, nullptr).connection_); + nullptr)); + EXPECT_EQ(nullptr, cluster_manager_->tcpConnForCluster("cluster_1", nullptr).connection_); + EXPECT_EQ(nullptr, cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, + &example_com_context)); EXPECT_EQ(nullptr, - cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr, - transport_socket_options_example_com)); - EXPECT_EQ(nullptr, - cluster_manager_ - ->tcpConnForCluster("cluster_1", nullptr, transport_socket_options_example_com) - .connection_); + cluster_manager_->tcpConnForCluster("cluster_1", &ibm_com_context).connection_); + EXPECT_EQ(nullptr, cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, + &ibm_com_context)); EXPECT_EQ(nullptr, - cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr, - transport_socket_options_ibm_com)); - EXPECT_EQ(nullptr, cluster_manager_ - ->tcpConnForCluster("cluster_1", nullptr, transport_socket_options_ibm_com) - .connection_); + cluster_manager_->tcpConnForCluster("cluster_1", &ibm_com_context).connection_); EXPECT_EQ(7UL, factory_.stats_.counter("cluster.cluster_1.upstream_cx_none_healthy").value()); @@ -2096,32 +2083,28 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemoveWithTls) { .WillRepeatedly(ReturnNew()); // This should provide us a CP for each of the above hosts, and for different SNIs - Tcp::ConnectionPool::MockInstance* tcp1 = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp2 = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp1_high = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::High, nullptr, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp2_high = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::High, nullptr, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp1 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp2 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp1_high = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::High, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp2_high = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::High, nullptr)); Tcp::ConnectionPool::MockInstance* tcp1_example_com = dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, transport_socket_options_example_com)); + "cluster_1", ResourcePriority::Default, &example_com_context)); Tcp::ConnectionPool::MockInstance* tcp2_example_com = dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, transport_socket_options_example_com)); + "cluster_1", ResourcePriority::Default, &example_com_context)); Tcp::ConnectionPool::MockInstance* tcp1_ibm_com = dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, transport_socket_options_ibm_com)); + "cluster_1", ResourcePriority::Default, &ibm_com_context)); Tcp::ConnectionPool::MockInstance* tcp2_ibm_com = dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, transport_socket_options_ibm_com)); + "cluster_1", ResourcePriority::Default, &ibm_com_context)); EXPECT_NE(tcp1, tcp2); EXPECT_NE(tcp1_high, tcp2_high); @@ -2192,19 +2175,17 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemoveWithTls) { EXPECT_EQ(cp2, cp3); EXPECT_EQ(cp2_high, cp3_high); - Tcp::ConnectionPool::MockInstance* tcp3 = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp3_high = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::High, nullptr, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp3 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp3_high = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::High, nullptr)); Tcp::ConnectionPool::MockInstance* tcp3_example_com = dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, transport_socket_options_example_com)); + "cluster_1", ResourcePriority::Default, &example_com_context)); Tcp::ConnectionPool::MockInstance* tcp3_ibm_com = dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, transport_socket_options_ibm_com)); + "cluster_1", ResourcePriority::Default, &ibm_com_context)); EXPECT_EQ(tcp2, tcp3); EXPECT_EQ(tcp2_high, tcp3_high); @@ -2268,9 +2249,8 @@ TEST_F(ClusterManagerImplTest, DynamicHostRemoveDefaultPriority) { dynamic_cast(cluster_manager_->httpConnPoolForCluster( "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); // Immediate drain, since this can happen with the HTTP codecs. EXPECT_CALL(*cp, addDrainedCallback(_)) @@ -2348,9 +2328,8 @@ TEST_F(ClusterManagerImplTest, ConnPoolDestroyWithDraining) { dynamic_cast(cluster_manager_->httpConnPoolForCluster( "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); // Remove the first host, this should lead to the cp being drained. Http::ConnectionPool::Instance::DrainedCb drained_cb; @@ -2396,9 +2375,8 @@ TEST_F(ClusterManagerImplTest, OriginalDstInitialization) { EXPECT_EQ(nullptr, cluster_manager_->httpConnPoolForCluster( "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); EXPECT_EQ(nullptr, cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, - nullptr, nullptr)); - EXPECT_EQ(nullptr, - cluster_manager_->tcpConnForCluster("cluster_1", nullptr, nullptr).connection_); + nullptr)); + EXPECT_EQ(nullptr, cluster_manager_->tcpConnForCluster("cluster_1", nullptr).connection_); EXPECT_EQ(3UL, factory_.stats_.counter("cluster.cluster_1.upstream_cx_none_healthy").value()); factory_.tls_.shutdownThread(); @@ -2898,7 +2876,7 @@ TEST_F(ClusterManagerImplTest, AddUpstreamFilters) { EXPECT_CALL(*connection, addFilter(_)).Times(0); EXPECT_CALL(factory_.tls_.dispatcher_, createClientConnection_(_, _, _, _)) .WillOnce(Return(connection)); - auto conn_data = cluster_manager_->tcpConnForCluster("cluster_1", nullptr, nullptr); + auto conn_data = cluster_manager_->tcpConnForCluster("cluster_1", nullptr); EXPECT_EQ(connection, conn_data.connection_.get()); factory_.tls_.shutdownThread(); } @@ -3136,7 +3114,7 @@ class SockoptsTest : public ClusterManagerImplTest { } return connection_; })); - cluster_manager_->tcpConnForCluster("SockoptsCluster", nullptr, nullptr); + cluster_manager_->tcpConnForCluster("SockoptsCluster", nullptr); } void expectSetsockoptFreebind() { @@ -3155,7 +3133,7 @@ class SockoptsTest : public ClusterManagerImplTest { EXPECT_EQ(nullptr, options.get()); return connection_; })); - auto conn_data = cluster_manager_->tcpConnForCluster("SockoptsCluster", nullptr, nullptr); + auto conn_data = cluster_manager_->tcpConnForCluster("SockoptsCluster", nullptr); EXPECT_EQ(connection_, conn_data.connection_.get()); } @@ -3370,7 +3348,7 @@ class TcpKeepaliveTest : public ClusterManagerImplTest { options, socket, envoy::api::v2::core::SocketOption::STATE_PREBIND))); return connection_; })); - cluster_manager_->tcpConnForCluster("TcpKeepaliveCluster", nullptr, nullptr); + cluster_manager_->tcpConnForCluster("TcpKeepaliveCluster", nullptr); return; } NiceMock os_sys_calls; @@ -3419,7 +3397,7 @@ class TcpKeepaliveTest : public ClusterManagerImplTest { return 0; })); } - auto conn_data = cluster_manager_->tcpConnForCluster("TcpKeepaliveCluster", nullptr, nullptr); + auto conn_data = cluster_manager_->tcpConnForCluster("TcpKeepaliveCluster", nullptr); EXPECT_EQ(connection_, conn_data.connection_.get()); } @@ -3433,7 +3411,7 @@ class TcpKeepaliveTest : public ClusterManagerImplTest { EXPECT_EQ(nullptr, options.get()); return connection_; })); - auto conn_data = cluster_manager_->tcpConnForCluster("TcpKeepaliveCluster", nullptr, nullptr); + auto conn_data = cluster_manager_->tcpConnForCluster("TcpKeepaliveCluster", nullptr); EXPECT_EQ(connection_, conn_data.connection_.get()); } @@ -3564,9 +3542,8 @@ TEST_F(ClusterManagerImplTest, ConnPoolsDrainedOnHostSetChange) { EXPECT_EQ(nullptr, cluster_manager_->httpConnPoolForCluster( "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); EXPECT_EQ(nullptr, cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, - nullptr, nullptr)); - EXPECT_EQ(nullptr, - cluster_manager_->tcpConnForCluster("cluster_1", nullptr, nullptr).connection_); + nullptr)); + EXPECT_EQ(nullptr, cluster_manager_->tcpConnForCluster("cluster_1", nullptr).connection_); Cluster& cluster = cluster_manager_->activeClusters().begin()->second; @@ -3603,13 +3580,11 @@ TEST_F(ClusterManagerImplTest, ConnPoolsDrainedOnHostSetChange) { dynamic_cast(cluster_manager_->httpConnPoolForCluster( "cluster_1", ResourcePriority::Default, Http::Protocol::Http2, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp1 = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp1 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp2 = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp2 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); EXPECT_NE(cp1, cp2); EXPECT_NE(tcp1, tcp2); @@ -3638,8 +3613,8 @@ TEST_F(ClusterManagerImplTest, ConnPoolsDrainedOnHostSetChange) { cp1 = dynamic_cast(cluster_manager_->httpConnPoolForCluster( "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); - tcp1 = dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + tcp1 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); HostSharedPtr host3 = makeTestHost(cluster.info(), "tcp://127.0.0.1:82"); @@ -3705,9 +3680,8 @@ TEST_F(ClusterManagerImplTest, ConnPoolsNotDrainedOnHostSetChange) { dynamic_cast(cluster_manager_->httpConnPoolForCluster( "cluster_1", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); - Tcp::ConnectionPool::MockInstance* tcp1 = - dynamic_cast(cluster_manager_->tcpConnPoolForCluster( - "cluster_1", ResourcePriority::Default, nullptr, nullptr)); + Tcp::ConnectionPool::MockInstance* tcp1 = dynamic_cast( + cluster_manager_->tcpConnPoolForCluster("cluster_1", ResourcePriority::Default, nullptr)); HostSharedPtr host2 = makeTestHost(cluster.info(), "tcp://127.0.0.1:82"); HostVector hosts_added; diff --git a/test/extensions/filters/network/dubbo_proxy/router_test.cc b/test/extensions/filters/network/dubbo_proxy/router_test.cc index 77f8df258809..e37c9f12bbb9 100644 --- a/test/extensions/filters/network/dubbo_proxy/router_test.cc +++ b/test/extensions/filters/network/dubbo_proxy/router_test.cc @@ -307,7 +307,7 @@ TEST_F(DubboRouterTest, NoHealthyHosts) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(context_.cluster_manager_, tcpConnPoolForCluster(cluster_name_, _, _, _)) + EXPECT_CALL(context_.cluster_manager_, tcpConnPoolForCluster(cluster_name_, _, _)) .WillOnce(Return(nullptr)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) diff --git a/test/extensions/filters/network/thrift_proxy/router_test.cc b/test/extensions/filters/network/thrift_proxy/router_test.cc index 6e6743016fe2..91134ff6a0e3 100644 --- a/test/extensions/filters/network/thrift_proxy/router_test.cc +++ b/test/extensions/filters/network/thrift_proxy/router_test.cc @@ -479,7 +479,7 @@ TEST_F(ThriftRouterTest, NoHealthyHosts) { EXPECT_CALL(callbacks_, route()).WillOnce(Return(route_ptr_)); EXPECT_CALL(*route_, routeEntry()).WillOnce(Return(&route_entry_)); EXPECT_CALL(route_entry_, clusterName()).WillRepeatedly(ReturnRef(cluster_name_)); - EXPECT_CALL(context_.cluster_manager_, tcpConnPoolForCluster(cluster_name_, _, _, _)) + EXPECT_CALL(context_.cluster_manager_, tcpConnPoolForCluster(cluster_name_, _, _)) .WillOnce(Return(nullptr)); EXPECT_CALL(callbacks_, sendLocalReply(_, _)) diff --git a/test/integration/tcp_conn_pool_integration_test.cc b/test/integration/tcp_conn_pool_integration_test.cc index 84ee8287c602..ce771247daa1 100644 --- a/test/integration/tcp_conn_pool_integration_test.cc +++ b/test/integration/tcp_conn_pool_integration_test.cc @@ -26,7 +26,7 @@ class TestFilter : public Network::ReadFilter { UNREFERENCED_PARAMETER(end_stream); Tcp::ConnectionPool::Instance* pool = cluster_manager_.tcpConnPoolForCluster( - "cluster_0", Upstream::ResourcePriority::Default, nullptr, nullptr); + "cluster_0", Upstream::ResourcePriority::Default, nullptr); ASSERT(pool != nullptr); requests_.emplace_back(*this, data); diff --git a/test/mocks/upstream/load_balancer_context.h b/test/mocks/upstream/load_balancer_context.h index 6d9e7046545c..5aaed4881791 100644 --- a/test/mocks/upstream/load_balancer_context.h +++ b/test/mocks/upstream/load_balancer_context.h @@ -19,6 +19,7 @@ class MockLoadBalancerContext : public LoadBalancerContext { MOCK_METHOD1(shouldSelectAnotherHost, bool(const Host&)); MOCK_CONST_METHOD0(hostSelectionRetryCount, uint32_t()); MOCK_CONST_METHOD0(upstreamSocketOptions, Network::Socket::OptionsSharedPtr()); + MOCK_CONST_METHOD0(upstreamTransportSocketOptions, Network::TransportSocketOptionsSharedPtr()); private: HealthyAndDegradedLoad priority_load_; diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 7d629b106108..4141474ec23e 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -135,7 +135,7 @@ MockClusterManager::MockClusterManager(TimeSource&) : MockClusterManager() {} MockClusterManager::MockClusterManager() { ON_CALL(*this, httpConnPoolForCluster(_, _, _, _)).WillByDefault(Return(&conn_pool_)); - ON_CALL(*this, tcpConnPoolForCluster(_, _, _, _)).WillByDefault(Return(&tcp_conn_pool_)); + ON_CALL(*this, tcpConnPoolForCluster(_, _, _)).WillByDefault(Return(&tcp_conn_pool_)); ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault(ReturnRef(async_client_)); ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault((ReturnRef(async_client_))); ON_CALL(*this, bindConfig()).WillByDefault(ReturnRef(bind_config_)); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 1b7f0b07b237..66dcb05380c1 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -292,8 +292,7 @@ class MockClusterManager : public ClusterManager { } Host::CreateConnectionData tcpConnForCluster(const std::string& cluster, - LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr) override { + LoadBalancerContext* context) override { MockHost::MockCreateConnectionData data = tcpConnForCluster_(cluster, context); return {Network::ClientConnectionPtr{data.connection_}, data.host_description_}; } @@ -310,11 +309,9 @@ class MockClusterManager : public ClusterManager { Http::ConnectionPool::Instance*(const std::string& cluster, ResourcePriority priority, Http::Protocol protocol, LoadBalancerContext* context)); - MOCK_METHOD4(tcpConnPoolForCluster, - Tcp::ConnectionPool::Instance*( - const std::string& cluster, ResourcePriority priority, - LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr transport_socket_options)); + MOCK_METHOD3(tcpConnPoolForCluster, + Tcp::ConnectionPool::Instance*(const std::string& cluster, ResourcePriority priority, + LoadBalancerContext* context)); MOCK_METHOD2(tcpConnForCluster_, MockHost::MockCreateConnectionData(const std::string& cluster, LoadBalancerContext* context)); diff --git a/test/server/config_validation/cluster_manager_test.cc b/test/server/config_validation/cluster_manager_test.cc index 0653c0cc8ae7..e7829296d5f7 100644 --- a/test/server/config_validation/cluster_manager_test.cc +++ b/test/server/config_validation/cluster_manager_test.cc @@ -54,7 +54,7 @@ TEST(ValidationClusterManagerTest, MockedMethods) { ClusterManagerPtr cluster_manager = factory.clusterManagerFromProto(bootstrap); EXPECT_EQ(nullptr, cluster_manager->httpConnPoolForCluster("cluster", ResourcePriority::Default, Http::Protocol::Http11, nullptr)); - Host::CreateConnectionData data = cluster_manager->tcpConnForCluster("cluster", nullptr, nullptr); + Host::CreateConnectionData data = cluster_manager->tcpConnForCluster("cluster", nullptr); EXPECT_EQ(nullptr, data.connection_); EXPECT_EQ(nullptr, data.host_description_); From 429d6c424f8c2f77af8633761e3ac8b8deeba21b Mon Sep 17 00:00:00 2001 From: Anatoly Scheglov Date: Wed, 2 Oct 2019 00:20:47 +0300 Subject: [PATCH 28/32] http conn man: stopped filters can add data from encodeTrailers (#8404) Now filters can return StopIteration from encodeHeaders, and then call addEncodedData from encodeTrailers. This allows the JsonTranscoderFilter to properly transcode gRPC status in trailer headers into a JSON reply body. Risk Level: Low Testing: unit-tests, integration tests in gRPC JSON transcoder filter. Documentation: n/a Release notes: n/a Fixes #8402 Signed-off-by: Anatoly Scheglov --- include/envoy/http/filter.h | 6 +- source/common/http/conn_manager_impl.cc | 6 +- .../json_transcoder_filter.cc | 6 - test/common/http/conn_manager_impl_test.cc | 124 ++++++++++++++++++ .../grpc_json_transcoder_integration_test.cc | 29 +++- .../json_transcoder_filter_test.cc | 54 ++++++++ 6 files changed, 213 insertions(+), 12 deletions(-) diff --git a/include/envoy/http/filter.h b/include/envoy/http/filter.h index 116506541d9e..e3a50d7ec2cd 100644 --- a/include/envoy/http/filter.h +++ b/include/envoy/http/filter.h @@ -242,7 +242,8 @@ class StreamDecoderFilterCallbacks : public virtual StreamFilterCallbacks { * * 4) If additional data needs to be added in the decodeTrailers() callback, this method can be * called in the context of the callback. All further filters will receive decodeData(..., false) - * followed by decodeTrailers(). + * followed by decodeTrailers(). However if the iteration is stopped, the added data will + * buffered, so that the further filters will not receive decodeData() before decodeHeaders(). * * It is an error to call this method in any other case. * @@ -562,7 +563,8 @@ class StreamEncoderFilterCallbacks : public virtual StreamFilterCallbacks { * * 4) If additional data needs to be added in the encodeTrailers() callback, this method can be * called in the context of the callback. All further filters will receive encodeData(..., false) - * followed by encodeTrailers(). + * followed by encodeTrailers(). However if the iteration is stopped, the added data will + * buffered, so that the further filters will not receive encodeData() before encodeHeaders(). * * It is an error to call this method in any other case. * diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 7285309ed21a..2e7a87edf573 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1085,7 +1085,8 @@ void ConnectionManagerImpl::ActiveStream::addDecodedData(ActiveStreamDecoderFilt Buffer::Instance& data, bool streaming) { if (state_.filter_call_state_ == 0 || (state_.filter_call_state_ & FilterCallState::DecodeHeaders) || - (state_.filter_call_state_ & FilterCallState::DecodeData)) { + (state_.filter_call_state_ & FilterCallState::DecodeData) || + ((state_.filter_call_state_ & FilterCallState::DecodeTrailers) && !filter.canIterate())) { // Make sure if this triggers watermarks, the correct action is taken. state_.decoder_filters_streaming_ = streaming; // If no call is happening or we are in the decode headers/data callback, buffer the data. @@ -1556,7 +1557,8 @@ void ConnectionManagerImpl::ActiveStream::addEncodedData(ActiveStreamEncoderFilt Buffer::Instance& data, bool streaming) { if (state_.filter_call_state_ == 0 || (state_.filter_call_state_ & FilterCallState::EncodeHeaders) || - (state_.filter_call_state_ & FilterCallState::EncodeData)) { + (state_.filter_call_state_ & FilterCallState::EncodeData) || + ((state_.filter_call_state_ & FilterCallState::EncodeTrailers) && !filter.canIterate())) { // Make sure if this triggers watermarks, the correct action is taken. state_.encoder_filters_streaming_ = streaming; // If no call is happening or we are in the decode headers/data callback, buffer the data. diff --git a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc index 6b9960218fa2..65419ec3125d 100644 --- a/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc +++ b/source/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter.cc @@ -563,12 +563,6 @@ bool JsonTranscoderFilter::maybeConvertGrpcStatus(Grpc::Status::GrpcStatus grpc_ return false; } - // We do not support responses with a separate trailer frame. - // TODO(ascheglov): remove this if after HCM can buffer data added from |encodeTrailers|. - if (response_headers_ != &trailers) { - return false; - } - // Send a serialized status only if there was no body. if (has_body_) { return false; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index f1e396ea7ac4..f3746ae40291 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -2917,6 +2917,130 @@ TEST_F(HttpConnectionManagerImplTest, FilterAddBodyInTrailersCallback) { HeaderMapPtr{new TestHeaderMapImpl{{"some", "trailer"}}}); } +// Don't send data frames, only headers and trailers. +TEST_F(HttpConnectionManagerImplTest, FilterAddBodyInTrailersCallback_NoDataFrames) { + InSequence s; + setup(false, ""); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{ + new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; + decoder->decodeHeaders(std::move(headers), false); + + HeaderMapPtr trailers{new TestHeaderMapImpl{{"foo", "bar"}}}; + decoder->decodeTrailers(std::move(trailers)); + })); + + setupFilterChain(2, 1); + + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + + Buffer::OwnedImpl trailers_data("hello"); + EXPECT_CALL(*decoder_filters_[0], decodeTrailers(_)) + .WillOnce(InvokeWithoutArgs([&]() -> FilterTrailersStatus { + decoder_filters_[0]->callbacks_->addDecodedData(trailers_data, false); + return FilterTrailersStatus::Continue; + })); + EXPECT_CALL(*decoder_filters_[0], decodeComplete()); + EXPECT_CALL(*decoder_filters_[1], decodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + EXPECT_CALL(*decoder_filters_[1], decodeData(_, false)) + .WillOnce(Return(FilterDataStatus::StopIterationAndBuffer)); + EXPECT_CALL(*decoder_filters_[1], decodeTrailers(_)) + .WillOnce(Return(FilterTrailersStatus::StopIteration)); + EXPECT_CALL(*decoder_filters_[1], decodeComplete()); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + + EXPECT_CALL(*encoder_filters_[0], encodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + decoder_filters_[0]->callbacks_->encodeHeaders( + HeaderMapPtr{new TestHeaderMapImpl{{":status", "200"}}}, false); + + EXPECT_CALL(*encoder_filters_[0], encodeTrailers(_)) + .WillOnce(InvokeWithoutArgs([&]() -> FilterTrailersStatus { + encoder_filters_[0]->callbacks_->addEncodedData(trailers_data, false); + return FilterTrailersStatus::Continue; + })); + EXPECT_CALL(*encoder_filters_[0], encodeComplete()); + EXPECT_CALL(response_encoder_, encodeHeaders(_, false)); + EXPECT_CALL(response_encoder_, encodeData(_, false)); + EXPECT_CALL(response_encoder_, encodeTrailers(_)); + expectOnDestroy(); + + decoder_filters_[0]->callbacks_->encodeTrailers( + HeaderMapPtr{new TestHeaderMapImpl{{"some", "trailer"}}}); +} + +// Don't send data frames, only headers and trailers. +TEST_F(HttpConnectionManagerImplTest, FilterAddBodyInTrailersCallback_ContinueAfterCallback) { + InSequence s; + setup(false, ""); + + EXPECT_CALL(*codec_, dispatch(_)).WillOnce(Invoke([&](Buffer::Instance&) -> void { + StreamDecoder* decoder = &conn_manager_->newStream(response_encoder_); + HeaderMapPtr headers{ + new TestHeaderMapImpl{{":authority", "host"}, {":path", "/"}, {":method", "GET"}}}; + decoder->decodeHeaders(std::move(headers), false); + + HeaderMapPtr trailers{new TestHeaderMapImpl{{"foo", "bar"}}}; + decoder->decodeTrailers(std::move(trailers)); + })); + + setupFilterChain(2, 1); + + EXPECT_CALL(*decoder_filters_[0], decodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + + Buffer::OwnedImpl trailers_data("hello"); + EXPECT_CALL(*decoder_filters_[0], decodeTrailers(_)) + .WillOnce(InvokeWithoutArgs([&]() -> FilterTrailersStatus { + decoder_filters_[0]->callbacks_->addDecodedData(trailers_data, false); + return FilterTrailersStatus::StopIteration; + })); + EXPECT_CALL(*decoder_filters_[0], decodeComplete()); + + // Kick off the incoming data. + Buffer::OwnedImpl fake_input("1234"); + conn_manager_->onData(fake_input, false); + + EXPECT_CALL(*decoder_filters_[1], decodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + EXPECT_CALL(*decoder_filters_[1], decodeData(_, false)) + .WillOnce(Return(FilterDataStatus::StopIterationAndBuffer)); + EXPECT_CALL(*decoder_filters_[1], decodeTrailers(_)) + .WillOnce(Return(FilterTrailersStatus::StopIteration)); + EXPECT_CALL(*decoder_filters_[1], decodeComplete()); + + decoder_filters_[0]->callbacks_->continueDecoding(); + + EXPECT_CALL(*encoder_filters_[0], encodeHeaders(_, false)) + .WillOnce(Return(FilterHeadersStatus::StopIteration)); + decoder_filters_[0]->callbacks_->encodeHeaders( + HeaderMapPtr{new TestHeaderMapImpl{{":status", "200"}}}, false); + + EXPECT_CALL(*encoder_filters_[0], encodeTrailers(_)) + .WillOnce(InvokeWithoutArgs([&]() -> FilterTrailersStatus { + encoder_filters_[0]->callbacks_->addEncodedData(trailers_data, false); + return FilterTrailersStatus::StopIteration; + })); + EXPECT_CALL(*encoder_filters_[0], encodeComplete()); + + decoder_filters_[0]->callbacks_->encodeTrailers( + HeaderMapPtr{new TestHeaderMapImpl{{"some", "trailer"}}}); + + EXPECT_CALL(response_encoder_, encodeHeaders(_, false)); + EXPECT_CALL(response_encoder_, encodeData(_, false)); + EXPECT_CALL(response_encoder_, encodeTrailers(_)); + expectOnDestroy(); + + encoder_filters_[0]->callbacks_->continueEncoding(); +} + // Add*Data during the *Data callbacks. TEST_F(HttpConnectionManagerImplTest, FilterAddBodyDuringDecodeData) { InSequence s; diff --git a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc index b4110b5b9acf..d6995eebd2fd 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/grpc_json_transcoder_integration_test.cc @@ -60,7 +60,8 @@ class GrpcJsonTranscoderIntegrationTest const std::vector& grpc_request_messages, const std::vector& grpc_response_messages, const Status& grpc_status, Http::HeaderMap&& response_headers, - const std::string& response_body, bool full_response = true) { + const std::string& response_body, bool full_response = true, + bool always_send_trailers = false) { codec_client_ = makeHttpConnection(lookupPort("http")); IntegrationStreamDecoderPtr response; @@ -98,7 +99,7 @@ class GrpcJsonTranscoderIntegrationTest Http::TestHeaderMapImpl response_headers; response_headers.insertStatus().value(200); response_headers.insertContentType().value(std::string("application/grpc")); - if (grpc_response_messages.empty()) { + if (grpc_response_messages.empty() && !always_send_trailers) { response_headers.insertGrpcStatus().value(static_cast(grpc_status.error_code())); response_headers.insertGrpcMessage().value(absl::string_view( grpc_status.error_message().data(), grpc_status.error_message().size())); @@ -375,6 +376,30 @@ TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryErrorConvertedToJson) { R"({"code":5,"message":"Shelf 100 Not Found"})"); } +// Upstream sends headers (e.g. sends metadata), and then sends trailer with an error. +TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryErrorInTrailerConvertedToJson) { + const std::string filter = + R"EOF( + name: envoy.grpc_json_transcoder + config: + proto_descriptor: "{}" + services: "bookstore.Bookstore" + convert_grpc_status: true + )EOF"; + config_helper_.addFilter( + fmt::format(filter, TestEnvironment::runfilesPath("/test/proto/bookstore.descriptor"))); + HttpIntegrationTest::initialize(); + testTranscoding( + Http::TestHeaderMapImpl{ + {":method", "GET"}, {":path", "/shelves/100"}, {":authority", "host"}}, + "", {"shelf: 100"}, {}, Status(Code::NOT_FOUND, "Shelf 100 Not Found"), + Http::TestHeaderMapImpl{{":status", "404"}, + {"content-type", "application/json"}, + {"grpc-status", UnexpectedHeaderValue}, + {"grpc-message", UnexpectedHeaderValue}}, + R"({"code":5,"message":"Shelf 100 Not Found"})", true, true); +} + TEST_P(GrpcJsonTranscoderIntegrationTest, UnaryDelete) { HttpIntegrationTest::initialize(); testTranscoding( diff --git a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc index f661f4609b42..006891df19f6 100644 --- a/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc +++ b/test/extensions/filters/http/grpc_json_transcoder/json_transcoder_filter_test.cc @@ -833,6 +833,60 @@ TEST_F(GrpcJsonTranscoderFilterConvertGrpcStatusTest, EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_.encodeHeaders(response_headers, true)); } +// Response with a header frame and a trailer frame. +// (E.g. a gRPC server sends metadata and then it sends an error.) +TEST_F(GrpcJsonTranscoderFilterConvertGrpcStatusTest, TranscodingStatusFromTrailer) { + Http::TestHeaderMapImpl response_headers{{"content-type", "application/grpc"}, + {":status", "200"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_.encodeHeaders(response_headers, false)); + EXPECT_EQ("application/json", response_headers.get_("content-type")); + + std::string expected_response(R"({"code":5,"message":"Resource not found"})"); + EXPECT_CALL(encoder_callbacks_, addEncodedData(_, false)) + .WillOnce(Invoke([&expected_response](Buffer::Instance& data, bool) { + EXPECT_EQ(expected_response, data.toString()); + })); + + Http::TestHeaderMapImpl response_trailers{ + {"grpc-status", "5"}, + {"grpc-message", "unused"}, + {"grpc-status-details-bin", "CAUSElJlc291cmNlIG5vdCBmb3VuZA"}}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.encodeTrailers(response_trailers)); + EXPECT_EQ("404", response_headers.get_(":status")); + EXPECT_EQ("application/json", response_headers.get_("content-type")); + EXPECT_FALSE(response_headers.has("grpc-status")); + EXPECT_FALSE(response_headers.has("grpc-message")); + EXPECT_FALSE(response_headers.has("grpc-status-details-bin")); +} + +// Server sends a response body, don't replace it. +TEST_F(GrpcJsonTranscoderFilterConvertGrpcStatusTest, SkipTranscodingStatusIfBodyIsPresent) { + Http::TestHeaderMapImpl response_headers{{"content-type", "application/grpc"}, + {":status", "200"}}; + + EXPECT_EQ(Http::FilterHeadersStatus::StopIteration, + filter_.encodeHeaders(response_headers, false)); + EXPECT_EQ("application/json", response_headers.get_("content-type")); + + bookstore::Shelf response; + response.set_id(20); + response.set_theme("Children"); + + auto response_data = Grpc::Common::serializeToGrpcFrame(response); + EXPECT_EQ(Http::FilterDataStatus::StopIterationAndBuffer, + filter_.encodeData(*response_data, false)); + + std::string response_json = response_data->toString(); + EXPECT_EQ(R"({"id":"20","theme":"Children"})", response_json); + + EXPECT_CALL(encoder_callbacks_, addEncodedData(_, _)).Times(0); + + Http::TestHeaderMapImpl response_trailers{{"grpc-status", "2"}, {"grpc-message", "not good"}}; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_.decodeTrailers(response_trailers)); +} + struct GrpcJsonTranscoderFilterPrintTestParam { std::string config_json_; std::string expected_response_; From abb14cdd7aa50d0a54a9288420a098314581782a Mon Sep 17 00:00:00 2001 From: Fred Douglas <43351173+fredlas@users.noreply.github.com> Date: Tue, 1 Oct 2019 17:22:08 -0400 Subject: [PATCH 29/32] config: full delta xDS (including ADS) support (#7293) Delta ADS/xDS support. Added NewGrpcMuxImpl, a GrpcMux implementation that supports delta (and is intended to soon take over for state-of-the-world). DeltaSubscriptionImpl (which can soon be renamed to/replace GrpcSubscriptionImpl) now uses that new class. NewGrpcMuxImpl is the first user of the recently added WatchMap. Risk Level: medium (huge new thing that people aren't using yet) Testing: added delta param variant to ads_integration_test Docs Changes: added source/common/config/README.md and accompanying diagram, added to root/configuration/overview.rst and root/intro/arch_overview/operations/dynamic_configuration.rst. Release Notes: added support for delta xDS (including ADS) delivery Fixes #4991 Signed-off-by: Fred Douglas --- api/xds_protocol.rst | 2 + .../configuration/overview/v2_overview.rst | 26 ++ .../operations/dynamic_configuration.rst | 12 + docs/root/intro/version_history.rst | 1 + include/envoy/config/BUILD | 17 -- include/envoy/config/grpc_mux.h | 40 +++ include/envoy/config/subscription.h | 2 +- include/envoy/config/subscription_factory.h | 2 +- include/envoy/config/xds_grpc_context.h | 42 ---- .../router/route_config_provider_manager.h | 2 +- include/envoy/server/listener_manager.h | 6 +- include/envoy/upstream/cluster_manager.h | 12 +- source/common/config/BUILD | 40 ++- source/common/config/README.md | 47 ++++ .../common/config/delta_subscription_impl.cc | 154 ++++++------ .../common/config/delta_subscription_impl.h | 109 ++++----- .../common/config/delta_subscription_state.cc | 98 +++----- .../common/config/delta_subscription_state.h | 41 ++-- .../config/filesystem_subscription_impl.cc | 2 +- .../config/filesystem_subscription_impl.h | 4 +- source/common/config/grpc_mux_impl.cc | 10 +- source/common/config/grpc_mux_impl.h | 27 ++- .../config/grpc_mux_subscription_impl.cc | 9 +- .../config/grpc_mux_subscription_impl.h | 6 +- source/common/config/grpc_stream.h | 2 +- source/common/config/grpc_subscription_impl.h | 15 +- .../common/config/http_subscription_impl.cc | 3 +- source/common/config/http_subscription_impl.h | 2 +- source/common/config/new_grpc_mux_impl.cc | 228 ++++++++++++++++++ source/common/config/new_grpc_mux_impl.h | 117 +++++++++ source/common/config/pausable_ack_queue.cc | 66 +++++ source/common/config/pausable_ack_queue.h | 43 ++++ .../config/subscription_factory_impl.cc | 29 ++- .../common/config/subscription_factory_impl.h | 4 +- source/common/config/xDS_code_diagram.png | Bin 0 -> 97291 bytes source/common/router/rds_impl.cc | 12 +- source/common/router/rds_impl.h | 8 +- source/common/router/scoped_rds.cc | 16 +- source/common/router/vhds.cc | 6 +- source/common/router/vhds.h | 4 +- source/common/runtime/runtime_impl.cc | 2 +- source/common/secret/sds_api.cc | 2 +- source/common/upstream/cds_api_impl.cc | 21 +- source/common/upstream/cds_api_impl.h | 9 +- .../common/upstream/cluster_manager_impl.cc | 88 +++++-- source/common/upstream/cluster_manager_impl.h | 10 +- source/common/upstream/eds.cc | 9 +- source/common/upstream/eds.h | 2 +- .../network/http_connection_manager/config.cc | 3 +- .../config_validation/cluster_manager.cc | 4 +- .../config_validation/cluster_manager.h | 2 +- source/server/config_validation/server.h | 9 +- source/server/lds_api.cc | 12 +- source/server/lds_api.h | 2 +- source/server/listener_manager_impl.h | 10 +- source/server/server.cc | 16 +- test/common/config/BUILD | 24 ++ .../config/delta_subscription_impl_test.cc | 74 ++++-- .../config/delta_subscription_state_test.cc | 109 ++++----- .../config/delta_subscription_test_harness.h | 43 +++- .../filesystem_subscription_test_harness.h | 15 +- test/common/config/grpc_mux_impl_test.cc | 14 +- test/common/config/grpc_stream_test.cc | 4 +- .../config/grpc_subscription_impl_test.cc | 10 +- .../config/grpc_subscription_test_harness.h | 15 +- .../config/http_subscription_test_harness.h | 4 +- test/common/config/new_grpc_mux_impl_test.cc | 111 +++++++++ .../config/subscription_factory_impl_test.cc | 2 +- test/common/config/subscription_impl_test.cc | 51 ++-- .../common/config/subscription_test_harness.h | 4 +- test/common/grpc/grpc_client_integration.h | 40 +-- test/common/router/rds_impl_test.cc | 14 +- test/common/router/scoped_rds_test.cc | 11 +- test/common/runtime/runtime_impl_test.cc | 4 +- test/common/upstream/cds_api_impl_test.cc | 6 +- .../upstream/cluster_manager_impl_test.cc | 12 +- test/common/upstream/eds_test.cc | 4 +- test/config/utility.cc | 37 ++- test/config/utility.h | 1 + test/integration/ads_integration.cc | 5 +- test/integration/ads_integration.h | 26 +- test/integration/ads_integration_test.cc | 95 +++++--- test/integration/cds_integration_test.cc | 3 +- test/integration/integration.cc | 94 +++++--- test/integration/integration.h | 15 +- test/integration/rtds_integration_test.cc | 7 +- .../scoped_rds_integration_test.cc | 4 +- test/mocks/config/BUILD | 1 - test/mocks/config/mocks.cc | 18 +- test/mocks/config/mocks.h | 20 +- test/mocks/router/mocks.h | 4 +- test/mocks/server/mocks.h | 11 +- test/mocks/upstream/mocks.cc | 2 +- test/mocks/upstream/mocks.h | 9 +- test/server/lds_api_test.cc | 3 +- test/server/listener_manager_impl_test.cc | 4 +- tools/spelling_dictionary.txt | 3 + 97 files changed, 1638 insertions(+), 762 deletions(-) delete mode 100644 include/envoy/config/xds_grpc_context.h create mode 100644 source/common/config/README.md create mode 100644 source/common/config/new_grpc_mux_impl.cc create mode 100644 source/common/config/new_grpc_mux_impl.h create mode 100644 source/common/config/pausable_ack_queue.cc create mode 100644 source/common/config/pausable_ack_queue.h create mode 100644 source/common/config/xDS_code_diagram.png create mode 100644 test/common/config/new_grpc_mux_impl_test.cc diff --git a/api/xds_protocol.rst b/api/xds_protocol.rst index 4d43306cdd1d..98211478e23e 100644 --- a/api/xds_protocol.rst +++ b/api/xds_protocol.rst @@ -367,6 +367,8 @@ An example minimal ``bootstrap.yaml`` fragment for ADS configuration is: admin: ... +.. _xds_protocol_delta: + Incremental xDS ~~~~~~~~~~~~~~~ diff --git a/docs/root/configuration/overview/v2_overview.rst b/docs/root/configuration/overview/v2_overview.rst index 3dcefb0067d5..b6b84fa95d44 100644 --- a/docs/root/configuration/overview/v2_overview.rst +++ b/docs/root/configuration/overview/v2_overview.rst @@ -570,6 +570,32 @@ to with the effect that the LDS stream will be directed to *some_ads_cluster* over the shared ADS channel. +.. _config_overview_v2_delta: + +Delta endpoints +--------------- + +The REST, filesystem, and original gRPC xDS implementations all deliver "state of the world" updates: +every CDS update must contain every cluster, with the absence of a cluster from an update implying +that the cluster is gone. For Envoy deployments with huge amounts of resources and even a trickle of +churn, these state-of-the-world updates can be cumbersome. + +As of 1.12.0, Envoy supports a "delta" variant of xDS (including ADS), where updates only contain +resources added/changed/removed. Delta xDS is a gRPC (only) protocol. Delta uses different +request/response protos than SotW (DeltaDiscovery{Request,Response}); see +:repo:`discovery.proto `. Conceptually, delta should be viewed as +a new xDS transport type: there is static, filesystem, REST, gRPC-SotW, and now gRPC-delta. +(Envoy's implementation of the gRPC-SotW/delta client happens to share most of its code between the +two, and something similar is likely possible on the server side. However, they are in fact +incompatible protocols. +:ref:`The specification of the delta xDS protocol's behavior is here `.) + +To use delta, simply set the api_type field of your +:ref:`ApiConfigSource ` proto(s) to DELTA_GRPC. +That works for both xDS and ADS; for ADS, it's the api_type field of +:ref:`DynamicResources.ads_config `, +as described in the previous section. + .. _config_overview_v2_mgmt_con_issues: Management Server Unreachability diff --git a/docs/root/intro/arch_overview/operations/dynamic_configuration.rst b/docs/root/intro/arch_overview/operations/dynamic_configuration.rst index bb51c6ebdf56..99bb323ed7c2 100644 --- a/docs/root/intro/arch_overview/operations/dynamic_configuration.rst +++ b/docs/root/intro/arch_overview/operations/dynamic_configuration.rst @@ -88,3 +88,15 @@ The :ref:`secret discovery service (SDS) ` laye by which Envoy can discover cryptographic secrets (certificate plus private key, TLS session ticket keys) for its listeners, as well as configuration of peer certificate validation logic (trusted root certs, revocations, etc). + +Aggregated xDS ("ADS") +----------------------------- + +EDS, CDS, etc. are each separate services, with different REST/gRPC service names, e.g. StreamListeners, StreamSecrets. For users looking to enforce the order in which resources of different types reach Envoy, there is aggregated xDS, a single gRPC service that carries all resource types in a single gRPC stream. (ADS is only supported by gRPC). :ref:`More details about ADS `. + +.. _arch_overview_dynamic_config_delta: + +Delta gRPC xDS +----------------------------- + +Standard xDS is "state-of-the-world": every CDS update must contain every cluster, with the absence of a cluster from an update implying that the cluster is gone. As of 1.12.0, Envoy supports a "delta" variant of xDS (including ADS), where updates only contain resources added/changed/removed. Delta xDS is a new protocol, with request/response protos different from SotW; existing control plane servers will need support added. :ref:`More details about delta `. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 3a7afd519c07..8c9a157748c5 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -11,6 +11,7 @@ Version history * admin: added config dump support for Secret Discovery Service :ref:`SecretConfigDump `. * api: added ::ref:`set_node_on_first_message_only ` option to omit the node identifier from the subsequent discovery requests on the same stream. * buffer filter: the buffer filter populates content-length header if not present, behavior can be disabled using the runtime feature `envoy.reloadable_features.buffer_filter_populate_content_length`. +* config: added support for :ref:`delta xDS ` (including ADS) delivery * config: enforcing that terminal filters (e.g. HttpConnectionManager for L4, router for L7) be the last in their respective filter chains. * config: added access log :ref:`extension filter`. * config: added support for :option:`--reject-unknown-dynamic-fields`, providing independent control diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 001d3cbe46af..7df8a214104f 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -65,20 +65,3 @@ envoy_cc_library( "//source/common/protobuf", ], ) - -envoy_cc_library( - name = "watch_map_interface", - hdrs = ["watch_map.h"], - deps = [ - ":subscription_interface", - ], -) - -envoy_cc_library( - name = "xds_grpc_context_interface", - hdrs = ["xds_grpc_context.h"], - deps = [ - ":subscription_interface", - "//source/common/protobuf", - ], -) diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 1d9920e2d5e3..1742ea60e060 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -67,6 +67,8 @@ class GrpcMuxWatch { using GrpcMuxWatchPtr = std::unique_ptr; +struct Watch; + /** * Manage one or more gRPC subscriptions on a single stream to management server. This can be used * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. @@ -111,6 +113,13 @@ class GrpcMux { */ virtual void resume(const std::string& type_url) PURE; + // For delta + virtual Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) PURE; + virtual void removeWatch(const std::string& type_url, Watch* watch) PURE; + /** * Retrieves the current pause state as set by pause()/resume(). * @param type_url type URL corresponding to xDS API, e.g. @@ -121,6 +130,37 @@ class GrpcMux { }; using GrpcMuxPtr = std::unique_ptr; +using GrpcMuxSharedPtr = std::shared_ptr; + +/** + * A grouping of callbacks that a GrpcMux should provide to its GrpcStream. + */ +template class GrpcStreamCallbacks { +public: + virtual ~GrpcStreamCallbacks() = default; + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to the + * gRPC stream having been successfully established. + */ + virtual void onStreamEstablished() PURE; + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to + * failure to establish the gRPC stream. + */ + virtual void onEstablishmentFailure() PURE; + + /** + * For the GrpcStream to pass received protos to the context. + */ + virtual void onDiscoveryResponse(std::unique_ptr&& message) PURE; + + /** + * For the GrpcStream to call when its rate limiting logic allows more requests to be sent. + */ + virtual void onWriteable() PURE; +}; } // namespace Config } // namespace Envoy diff --git a/include/envoy/config/subscription.h b/include/envoy/config/subscription.h index ee21da758d10..009c4a265dc6 100644 --- a/include/envoy/config/subscription.h +++ b/include/envoy/config/subscription.h @@ -89,7 +89,7 @@ class Subscription { * @param resources vector of resource names to fetch. It's a (not unordered_)set so that it can * be passed to std::set_difference, which must be given sorted collections. */ - virtual void updateResources(const std::set& update_to_these_names) PURE; + virtual void updateResourceInterest(const std::set& update_to_these_names) PURE; }; using SubscriptionPtr = std::unique_ptr; diff --git a/include/envoy/config/subscription_factory.h b/include/envoy/config/subscription_factory.h index b1733afda6c4..5268ac7f2abe 100644 --- a/include/envoy/config/subscription_factory.h +++ b/include/envoy/config/subscription_factory.h @@ -24,7 +24,7 @@ class SubscriptionFactory { virtual SubscriptionPtr subscriptionFromConfigSource(const envoy::api::v2::core::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, - SubscriptionCallbacks& callbacks) PURE; + SubscriptionCallbacks& callbacks, bool is_delta) PURE; }; } // namespace Config diff --git a/include/envoy/config/xds_grpc_context.h b/include/envoy/config/xds_grpc_context.h deleted file mode 100644 index 312d3e0650f7..000000000000 --- a/include/envoy/config/xds_grpc_context.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include "envoy/common/pure.h" -#include "envoy/config/subscription.h" - -#include "common/protobuf/protobuf.h" - -namespace Envoy { -namespace Config { - -/** - * A grouping of callbacks that an XdsGrpcContext should provide to its GrpcStream. - */ -template class GrpcStreamCallbacks { -public: - virtual ~GrpcStreamCallbacks() = default; - - /** - * For the GrpcStream to prompt the context to take appropriate action in response to the - * gRPC stream having been successfully established. - */ - virtual void onStreamEstablished() PURE; - - /** - * For the GrpcStream to prompt the context to take appropriate action in response to - * failure to establish the gRPC stream. - */ - virtual void onEstablishmentFailure() PURE; - - /** - * For the GrpcStream to pass received protos to the context. - */ - virtual void onDiscoveryResponse(std::unique_ptr&& message) PURE; - - /** - * For the GrpcStream to call when its rate limiting logic allows more requests to be sent. - */ - virtual void onWriteable() PURE; -}; - -} // namespace Config -} // namespace Envoy diff --git a/include/envoy/router/route_config_provider_manager.h b/include/envoy/router/route_config_provider_manager.h index 9912aa460356..accb99c85f24 100644 --- a/include/envoy/router/route_config_provider_manager.h +++ b/include/envoy/router/route_config_provider_manager.h @@ -39,7 +39,7 @@ class RouteConfigProviderManager { virtual RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Init::Manager& init_manager) PURE; + Init::Manager& init_manager, bool is_delta) PURE; /** * Get a RouteConfigSharedPtr for a statically defined route. Ownership is as described for diff --git a/include/envoy/server/listener_manager.h b/include/envoy/server/listener_manager.h index ce2a015decf3..54d4b4a9bbcb 100644 --- a/include/envoy/server/listener_manager.h +++ b/include/envoy/server/listener_manager.h @@ -39,7 +39,8 @@ class ListenerComponentFactory { * @return an LDS API provider. * @param lds_config supplies the management server configuration. */ - virtual LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) PURE; + virtual LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) PURE; /** * Creates a socket. @@ -128,7 +129,8 @@ class ListenerManager { * pieces of the server existing. * @param lds_config supplies the management server configuration. */ - virtual void createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) PURE; + virtual void createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) PURE; /** * @return std::vector> a list of the currently diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index d911f7982137..fd3255fc2dde 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -178,14 +178,14 @@ class ClusterManager { virtual const envoy::api::v2::core::BindConfig& bindConfig() const PURE; /** - * Return a reference to the singleton ADS provider for upstream control plane muxing of xDS. This - * is treated somewhat as a special case in ClusterManager, since it does not relate logically to - * the management of clusters but instead is required early in ClusterManager/server + * Returns a shared_ptr to the singleton xDS-over-gRPC provider for upstream control plane muxing + * of xDS. This is treated somewhat as a special case in ClusterManager, since it does not relate + * logically to the management of clusters but instead is required early in ClusterManager/server * initialization and in various sites that need ClusterManager for xDS API interfacing. * * @return GrpcMux& ADS API provider referencee. */ - virtual Config::GrpcMux& adsMux() PURE; + virtual Config::GrpcMuxSharedPtr adsMux() PURE; /** * @return Grpc::AsyncClientManager& the cluster manager's gRPC client manager. @@ -223,6 +223,8 @@ class ClusterManager { virtual Config::SubscriptionFactory& subscriptionFactory() PURE; virtual std::size_t warmingClusterCount() const PURE; + + virtual bool xdsIsDelta() const PURE; }; using ClusterManagerPtr = std::unique_ptr; @@ -295,7 +297,7 @@ class ClusterManagerFactory { /** * Create a CDS API provider from configuration proto. */ - virtual CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, + virtual CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, ClusterManager& cm) PURE; /** diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 5d346543180c..0f18f5524d00 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -71,8 +71,8 @@ envoy_cc_library( srcs = ["delta_subscription_impl.cc"], hdrs = ["delta_subscription_impl.h"], deps = [ - ":delta_subscription_state_lib", ":grpc_stream_lib", + ":new_grpc_mux_lib", ":utility_lib", "//include/envoy/config:subscription_interface", "//include/envoy/grpc:async_client_interface", @@ -90,6 +90,7 @@ envoy_cc_library( srcs = ["delta_subscription_state.cc"], hdrs = ["delta_subscription_state.h"], deps = [ + ":pausable_ack_queue_lib", "//include/envoy/config:subscription_interface", "//include/envoy/event:dispatcher_interface", "//source/common/common:assert_lib", @@ -125,7 +126,6 @@ envoy_cc_library( ":utility_lib", "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", - "//include/envoy/config:xds_grpc_context_interface", "//include/envoy/grpc:async_client_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/common:backoff_lib", @@ -217,6 +217,20 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "new_grpc_mux_lib", + srcs = ["new_grpc_mux_impl.cc"], + hdrs = ["new_grpc_mux_impl.h"], + deps = [ + ":delta_subscription_state_lib", + ":grpc_stream_lib", + ":pausable_ack_queue_lib", + ":watch_map_lib", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/grpc:async_client_interface", + ], +) + envoy_cc_library( name = "http_subscription_lib", srcs = ["http_subscription_impl.cc"], @@ -261,6 +275,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "pausable_ack_queue_lib", + srcs = ["pausable_ack_queue.cc"], + hdrs = ["pausable_ack_queue.h"], + deps = [ + "//source/common/common:assert_lib", + "@envoy_api//envoy/api/v2:discovery_cc", + ], +) + envoy_cc_library( name = "protobuf_link_hacks", hdrs = ["protobuf_link_hacks.h"], @@ -300,12 +324,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "resources_lib", - hdrs = ["resources.h"], - deps = ["//source/common/singleton:const_singleton"], -) - envoy_cc_library( name = "remote_data_fetcher_lib", srcs = ["remote_data_fetcher.cc"], @@ -320,6 +338,12 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "resources_lib", + hdrs = ["resources.h"], + deps = ["//source/common/singleton:const_singleton"], +) + envoy_cc_library( name = "runtime_utility_lib", srcs = ["runtime_utility.cc"], diff --git a/source/common/config/README.md b/source/common/config/README.md new file mode 100644 index 000000000000..7774c91befbf --- /dev/null +++ b/source/common/config/README.md @@ -0,0 +1,47 @@ +# xDS + +xDS stands for [fill in the blank] Discovery Service. It provides dynamic config discovery/updates. + +tldr: xDS can use the filesystem, REST, or gRPC. gRPC xDS comes in four flavors. +However, Envoy code uses all of that via the same Subscription interface. +If you are an Envoy developer with your hands on a valid Subscription object, +you can mostly forget the filesystem/REST/gRPC distinction, and you can +especially forget about the gRPC flavors. All of that is specified in the +bootstrap config, which is read and put into action by ClusterManagerImpl. + +Note that there can be multiple active gRPC subscriptions for a single resource +type. This concept is called "resource watches". If one EDS subscription +subscribes to X and Y, and another subscribes to Y and Z, the underlying +subscription logic will maintain a subscription to the union: X Y and Z. Updates +to X will be delivered to the first object, Y to both, Z to the second. This +logic is implemented by WatchMap. + +### If you are working on Envoy's gRPC xDS client logic itself, read on. + +When using gRPC, xDS has two pairs of options: aggregated/non-aggregated, and +delta/state-of-the-world updates. All four combinations of these are usable. + +"Aggregated" means that EDS, CDS, etc resources are all carried by the same gRPC stream. +For Envoy's implementation of xDS client logic, there is effectively no difference +between aggregated xDS and non-aggregated: they both use the same request/response protos. The +non-aggregated case is handled by running the aggregated logic, and just happening to only have 1 +xDS subscription type to "aggregate", i.e., NewGrpcMuxImpl only has one +DeltaSubscriptionState entry in its map. + +However, to the config server, there is a huge difference: when using ADS (caused +by the user providing an ads_config in the bootstrap config), the gRPC client sets +its method string to {Delta,Stream}AggregatedResources, as opposed to {Delta,Stream}Clusters, +{Delta,Stream}Routes, etc. So, despite using the same request/response protos, +and having identical client code, they're actually different gRPC services. + +Delta vs state-of-the-world is a question of wire format: the protos in question are named +[Delta]Discovery{Request,Response}. That is what the GrpcMux interface is useful for: its +NewGrpcMuxImpl (TODO may be renamed) implementation works with DeltaDiscovery{Request,Response} and has +delta-specific logic; its GrpxMuxImpl implementation (TODO will be merged into NewGrpcMuxImpl) works with Discovery{Request,Response} +and has SotW-specific logic. Both the delta and SotW Subscription implementations (TODO will be merged) hold a shared_ptr. +The shared_ptr allows for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. + +![xDS_code_diagram_june2019](xDS_code_diagram_june2019.png) + +Note that the orange flow does not necessarily have to happen in response to the blue flow; there can be spontaneous updates. ACKs are not shown in this diagram; they are also carred by the [Delta]DiscoveryRequest protos. +What does GrpcXdsContext even do in this diagram? Just own things and pass through function calls? Answer: it sequences the requests and ACKs that the various type_urls send. diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index 8bc69f04f4eb..ed34dac39bea 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -1,106 +1,94 @@ #include "common/config/delta_subscription_impl.h" -#include "common/common/assert.h" -#include "common/common/backoff_strategy.h" -#include "common/common/token_bucket_impl.h" -#include "common/config/utility.h" -#include "common/protobuf/protobuf.h" -#include "common/protobuf/utility.h" - namespace Envoy { namespace Config { -DeltaSubscriptionImpl::DeltaSubscriptionImpl( - const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, - Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - absl::string_view type_url, Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, SubscriptionCallbacks& callbacks, - SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout) - : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings), - type_url_(type_url), local_info_(local_info), callbacks_(callbacks), stats_(stats), - dispatcher_(dispatcher), init_fetch_timeout_(init_fetch_timeout) {} +DeltaSubscriptionImpl::DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, + SubscriptionCallbacks& callbacks, + SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout, + bool is_aggregated) + : context_(std::move(context)), type_url_(type_url), callbacks_(callbacks), stats_(stats), + init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) {} -void DeltaSubscriptionImpl::pause() { state_->pause(); } -void DeltaSubscriptionImpl::resume() { - state_->resume(); - trySendDiscoveryRequests(); +DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { + if (watch_) { + context_->removeWatch(type_url_, watch_); + } } -// Config::Subscription -void DeltaSubscriptionImpl::start(const std::set& resource_names) { - state_ = std::make_unique( - type_url_, resource_names, callbacks_, local_info_, init_fetch_timeout_, dispatcher_, stats_); - grpc_stream_.establishNewStream(); - updateResources(resource_names); -} +void DeltaSubscriptionImpl::pause() { context_->pause(type_url_); } -void DeltaSubscriptionImpl::updateResources(const std::set& update_to_these_names) { - state_->updateResourceInterest(update_to_these_names); - // Tell the server about our new interests, if there are any. - trySendDiscoveryRequests(); -} +void DeltaSubscriptionImpl::resume() { context_->resume(type_url_); } -// Config::GrpcStreamCallbacks -void DeltaSubscriptionImpl::onStreamEstablished() { - state_->markStreamFresh(); - trySendDiscoveryRequests(); +// Config::DeltaSubscription +void DeltaSubscriptionImpl::start(const std::set& resources) { + // ADS initial request batching relies on the users of the GrpcMux *not* calling start on it, + // whereas non-ADS xDS users must call it themselves. + if (!is_aggregated_) { + context_->start(); + } + watch_ = context_->addOrUpdateWatch(type_url_, watch_, resources, *this, init_fetch_timeout_); + stats_.update_attempt_.inc(); } -void DeltaSubscriptionImpl::onEstablishmentFailure() { state_->handleEstablishmentFailure(); } - -void DeltaSubscriptionImpl::onDiscoveryResponse( - std::unique_ptr&& message) { - ENVOY_LOG(debug, "Received gRPC message for {} at version {}", type_url_, - message->system_version_info()); - kickOffAck(state_->handleResponse(*message)); +void DeltaSubscriptionImpl::updateResourceInterest( + const std::set& update_to_these_names) { + watch_ = context_->addOrUpdateWatch(type_url_, watch_, update_to_these_names, *this, + init_fetch_timeout_); + stats_.update_attempt_.inc(); } -void DeltaSubscriptionImpl::onWriteable() { trySendDiscoveryRequests(); } - -void DeltaSubscriptionImpl::kickOffAck(UpdateAck ack) { - ack_queue_.push(ack); - trySendDiscoveryRequests(); +// Config::SubscriptionCallbacks +void DeltaSubscriptionImpl::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + stats_.update_attempt_.inc(); + callbacks_.onConfigUpdate(resources, version_info); + stats_.update_success_.inc(); + stats_.version_.set(HashUtil::xxHash64(version_info)); } -// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check -// whether we *want* to send a DeltaDiscoveryRequest). -bool DeltaSubscriptionImpl::canSendDiscoveryRequest() { - if (state_->paused()) { - ENVOY_LOG(trace, "API {} paused; discovery request on hold for now.", type_url_); - return false; - } else if (!grpc_stream_.grpcStreamAvailable()) { - ENVOY_LOG(trace, "No stream available to send a DiscoveryRequest for {}.", type_url_); - return false; - } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { - ENVOY_LOG(trace, "{} DiscoveryRequest hit rate limit; will try later.", type_url_); - return false; - } - return true; +void DeltaSubscriptionImpl::onConfigUpdate( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + stats_.update_attempt_.inc(); + callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); + stats_.update_success_.inc(); + stats_.version_.set(HashUtil::xxHash64(system_version_info)); } -// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or -// a subscription update. (Does not check whether we *can* send a DeltaDiscoveryRequest). -bool DeltaSubscriptionImpl::wantToSendDiscoveryRequest() { - return !ack_queue_.empty() || state_->subscriptionUpdatePending(); +void DeltaSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, + const EnvoyException* e) { + switch (reason) { + case Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure: + // This is a gRPC-stream-level establishment failure, not an xDS-protocol-level failure. + // So, don't onConfigUpdateFailed() here. Instead, allow a retry of the gRPC stream. + // If init_fetch_timeout_ is non-zero, the server will continue startup after that timeout. + stats_.update_failure_.inc(); + // TODO(fredlas) remove; it only makes sense to count start() and updateResourceInterest() + // as attempts. + stats_.update_attempt_.inc(); + break; + case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: + stats_.init_fetch_timeout_.inc(); + // TODO(fredlas) remove; it only makes sense to count start() and updateResourceInterest() + // as attempts. + stats_.update_attempt_.inc(); + callbacks_.onConfigUpdateFailed(reason, e); + break; + case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: + // We expect Envoy exception to be thrown when update is rejected. + ASSERT(e != nullptr); + stats_.update_rejected_.inc(); + callbacks_.onConfigUpdateFailed(reason, e); + break; + } } -void DeltaSubscriptionImpl::trySendDiscoveryRequests() { - while (wantToSendDiscoveryRequest() && canSendDiscoveryRequest()) { - envoy::api::v2::DeltaDiscoveryRequest request = state_->getNextRequest(); - if (!ack_queue_.empty()) { - const UpdateAck& ack = ack_queue_.front(); - request.set_response_nonce(ack.nonce_); - if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { - // Don't needlessly make the field present-but-empty if status is ok. - request.mutable_error_detail()->CopyFrom(ack.error_detail_); - } - ack_queue_.pop(); - } - ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url_, request.DebugString()); - grpc_stream_.sendMessage(request); - } - grpc_stream_.maybeUpdateQueueSizeStat(ack_queue_.size()); +std::string DeltaSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { + return callbacks_.resourceName(resource); } } // namespace Config diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index dbee51c688c3..b45be7c4da85 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -1,93 +1,70 @@ #pragma once -#include - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/token_bucket.h" #include "envoy/config/subscription.h" -#include "envoy/config/xds_grpc_context.h" -#include "envoy/local_info/local_info.h" -#include "common/common/logger.h" -#include "common/config/delta_subscription_state.h" -#include "common/config/grpc_stream.h" -#include "common/grpc/common.h" +#include "common/config/new_grpc_mux_impl.h" +#include "common/config/utility.h" namespace Envoy { namespace Config { -/** - * Manages the logic of a (non-aggregated) delta xDS subscription. - * TODO(fredlas) add aggregation support. The plan is for that to happen in XdsGrpcContext, - * which this class will then "have a" rather than "be a". - * TODO(kyessenov) implement skip_subsequent_node for delta xDS subscription. - */ -class DeltaSubscriptionImpl : public Subscription, - public GrpcStreamCallbacks, - public Logger::Loggable { +// DeltaSubscriptionImpl provides a top-level interface to the Envoy's gRPC communication with +// an xDS server, for use by the various xDS users within Envoy. It is built around a (shared) +// NewGrpcMuxImpl, and the further machinery underlying that. An xDS user indicates interest in +// various resources via start() and updateResourceInterest(). It receives updates to those +// resources via the SubscriptionCallbacks it provides. Multiple users can each have their own +// Subscription object for the same type_url; NewGrpcMuxImpl maintains a subscription to the +// union of interested resources, and delivers to the users just the resource updates that they +// are "watching" for. +// +// DeltaSubscriptionImpl and NewGrpcMuxImpl are both built to provide both regular xDS and ADS, +// distinguished by whether multiple DeltaSubscriptionImpls are sharing a single +// NewGrpcMuxImpl. (And by the gRPC method string, but that's taken care of over in +// SubscriptionFactory). +// +// Why does DeltaSubscriptionImpl itself implement the SubscriptionCallbacks interface? So that it +// can write to SubscriptionStats (which needs to live out here in the DeltaSubscriptionImpl) upon a +// config update. The idea is, DeltaSubscriptionImpl presents itself to WatchMap as the +// SubscriptionCallbacks, and then passes (after incrementing stats) all callbacks through to +// callbacks_, which are the real SubscriptionCallbacks. +class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks { public: - DeltaSubscriptionImpl(const LocalInfo::LocalInfo& local_info, - Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - absl::string_view type_url, Runtime::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + // is_aggregated: whether the underlying mux/context is providing ADS to us and others, or whether + // it's all ours. The practical difference is that we ourselves must call start() on it only in + // the latter case. + DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, SubscriptionCallbacks& callbacks, SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout); + std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); + ~DeltaSubscriptionImpl() override; void pause(); void resume(); // Config::Subscription void start(const std::set& resource_names) override; - void updateResources(const std::set& update_to_these_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; - // Config::GrpcStreamCallbacks - void onStreamEstablished() override; - void onEstablishmentFailure() override; - void - onDiscoveryResponse(std::unique_ptr&& message) override; + // Config::SubscriptionCallbacks (all pass through to callbacks_!) + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override; + void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) override; + std::string resourceName(const ProtobufWkt::Any& resource) override; - void onWriteable() override; + GrpcMuxSharedPtr getContextForTest() { return context_; } private: - void kickOffAck(UpdateAck ack); - - // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check - // whether we *want* to send a DeltaDiscoveryRequest). - bool canSendDiscoveryRequest(); - - // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or - // a subscription update. (Does not check whether we *can* send a DeltaDiscoveryRequest). - bool wantToSendDiscoveryRequest(); - - void trySendDiscoveryRequests(); - - GrpcStream - grpc_stream_; - + GrpcMuxSharedPtr context_; const std::string type_url_; - - // An item in the queue represents a DeltaDiscoveryRequest that must be sent. If an item is not - // empty, it is the ACK (nonce + error_detail) to set on that request. An empty entry should - // still send a request; it just won't have an ACK. - // - // More details: DeltaDiscoveryRequest plays two independent roles: - // 1) informing the server of what resources we're interested in, and - // 2) acknowledging resources the server has sent us. - // Each entry in this queue was added for exactly one of those purposes, but since the - // subscription interest is tracked separately, in a non-queue way, subscription changes can get - // mixed in with an ACK request. In that case, the entry that the subscription change originally - // queued up *does* still get sent, just empty and pointless. (TODO(fredlas) we would like to skip - // those no-op requests). - std::queue ack_queue_; - - const LocalInfo::LocalInfo& local_info_; SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; - Event::Dispatcher& dispatcher_; + // NOTE: if another subscription of the same type_url has already been started, this value will be + // ignored in favor of the other subscription's. std::chrono::milliseconds init_fetch_timeout_; - - std::unique_ptr state_; + Watch* watch_{}; + const bool is_aggregated_; }; } // namespace Config diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index 5c5a795b7af0..93ddc18164da 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -1,78 +1,44 @@ #include "common/config/delta_subscription_state.h" #include "common/common/assert.h" +#include "common/common/hash.h" namespace Envoy { namespace Config { -DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, - const std::set& resource_names, +DeltaSubscriptionState::DeltaSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, - SubscriptionStats& stats) - : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), - init_fetch_timeout_(init_fetch_timeout), stats_(stats) { - // In normal usage of updateResourceInterest(), the caller is supposed to cause a discovery - // request to be queued if it returns true. We don't need to do that because we know that the - // subscription gRPC stream is not yet established, and establishment causes a request. - updateResourceInterest(resource_names); - setInitFetchTimeout(dispatcher); -} - -void DeltaSubscriptionState::setInitFetchTimeout(Event::Dispatcher& dispatcher) { + Event::Dispatcher& dispatcher) + : type_url_(std::move(type_url)), callbacks_(callbacks), local_info_(local_info), + init_fetch_timeout_(init_fetch_timeout) { if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { - stats_.init_fetch_timeout_.inc(); ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); - callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, - nullptr); + callbacks_.onConfigUpdateFailed(ConfigUpdateFailureReason::FetchTimedout, nullptr); }); init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } } -void DeltaSubscriptionState::pause() { - ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url_); - ASSERT(!paused_); - paused_ = true; -} - -void DeltaSubscriptionState::resume() { - ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); - ASSERT(paused_); - paused_ = false; -} - -// Returns true if there is any meaningful change in our subscription interest, worth reporting to -// the server. -void DeltaSubscriptionState::updateResourceInterest( - const std::set& update_to_these_names) { - std::vector cur_added; - std::vector cur_removed; - - std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), - resource_names_.begin(), resource_names_.end(), - std::inserter(cur_added, cur_added.begin())); - std::set_difference(resource_names_.begin(), resource_names_.end(), update_to_these_names.begin(), - update_to_these_names.end(), std::inserter(cur_removed, cur_removed.begin())); - +void DeltaSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed) { for (const auto& a : cur_added) { setResourceWaitingForServer(a); - // Removed->added requires us to keep track of it as a "new" addition, since our user may have - // forgotten its copy of the resource after instructing us to remove it, and so needs to be - // reminded of it. + // If interest in a resource is removed-then-added (all before a discovery request + // can be sent), we must treat it as a "new" addition: our user may have forgotten its + // copy of the resource after instructing us to remove it, and need to be reminded of it. names_removed_.erase(a); names_added_.insert(a); } for (const auto& r : cur_removed) { setLostInterestInResource(r); - // Ideally, when a resource is added-then-removed in between requests, we would avoid putting - // a superfluous "unsubscribe [resource that was never subscribed]" in the request. However, - // the removed-then-added case *does* need to go in the request, and due to how we accomplish - // that, it's difficult to distinguish remove-add-remove from add-remove (because "remove-add" - // has to be treated as equivalent to just "add"). + // Ideally, when interest in a resource is added-then-removed in between requests, + // we would avoid putting a superfluous "unsubscribe [resource that was never subscribed]" + // in the request. However, the removed-then-added case *does* need to go in the request, + // and due to how we accomplish that, it's difficult to distinguish remove-add-remove from + // add-remove (because "remove-add" has to be treated as equivalent to just "add"). names_added_.erase(r); names_removed_.insert(r); } @@ -89,8 +55,7 @@ UpdateAck DeltaSubscriptionState::handleResponse(const envoy::api::v2::DeltaDiscoveryResponse& message) { // We *always* copy the response's nonce into the next request, even if we're going to make that // request a NACK by setting error_detail. - UpdateAck ack(message.nonce()); - stats_.update_attempt_.inc(); + UpdateAck ack(message.nonce(), type_url_); try { handleGoodResponse(message); } catch (const EnvoyException& e) { @@ -108,6 +73,12 @@ void DeltaSubscriptionState::handleGoodResponse( throw EnvoyException( fmt::format("duplicate name {} found among added/updated resources", resource.name())); } + if (message.type_url() != resource.resource().type_url()) { + throw EnvoyException(fmt::format("type URL {} embedded in an individual Any does not match " + "the message-wide type URL {} in DeltaDiscoveryResponse {}", + resource.resource().type_url(), message.type_url(), + message.DebugString())); + } } for (const auto& name : message.removed_resources()) { if (!names_added_removed.insert(name).second) { @@ -115,7 +86,6 @@ void DeltaSubscriptionState::handleGoodResponse( fmt::format("duplicate name {} found in the union of added+removed resources", name)); } } - callbacks_.onConfigUpdate(message.resources(), message.removed_resources(), message.system_version_info()); for (const auto& resource : message.resources()) { @@ -134,8 +104,6 @@ void DeltaSubscriptionState::handleGoodResponse( setResourceWaitingForServer(resource_name); } } - stats_.update_success_.inc(); - stats_.version_.set(HashUtil::xxHash64(message.system_version_info())); ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url_, message.resources().size(), message.removed_resources().size()); } @@ -145,19 +113,16 @@ void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAc ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); ack.error_detail_.set_message(e.what()); disableInitFetchTimeoutTimer(); - stats_.update_rejected_.inc(); ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } void DeltaSubscriptionState::handleEstablishmentFailure() { - // New gRPC stream will be established and send requests again. - // If init_fetch_timeout is non-zero, server will continue startup after it timeout - stats_.update_failure_.inc(); - stats_.update_attempt_.inc(); + callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + nullptr); } -envoy::api::v2::DeltaDiscoveryRequest DeltaSubscriptionState::getNextRequest() { +envoy::api::v2::DeltaDiscoveryRequest DeltaSubscriptionState::getNextRequestAckless() { envoy::api::v2::DeltaDiscoveryRequest request; if (!any_request_sent_yet_in_current_stream_) { any_request_sent_yet_in_current_stream_ = true; @@ -189,6 +154,17 @@ envoy::api::v2::DeltaDiscoveryRequest DeltaSubscriptionState::getNextRequest() { return request; } +envoy::api::v2::DeltaDiscoveryRequest +DeltaSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { + envoy::api::v2::DeltaDiscoveryRequest request = getNextRequestAckless(); + request.set_response_nonce(ack.nonce_); + if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { + // Don't needlessly make the field present-but-empty if status is ok. + request.mutable_error_detail()->CopyFrom(ack.error_detail_); + } + return request; +} + void DeltaSubscriptionState::disableInitFetchTimeoutTimer() { if (init_fetch_timeout_timer_) { init_fetch_timeout_timer_->disableTimer(); diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index f21e0b895b9e..6442d4b7a84f 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -7,34 +7,26 @@ #include "envoy/local_info/local_info.h" #include "common/common/assert.h" -#include "common/common/hash.h" #include "common/common/logger.h" +#include "common/config/pausable_ack_queue.h" namespace Envoy { namespace Config { -struct UpdateAck { - UpdateAck(absl::string_view nonce) : nonce_(nonce) {} - std::string nonce_; - ::google::rpc::Status error_detail_; -}; - -// Tracks the xDS protocol state of an individual ongoing delta xDS session. +// Tracks the xDS protocol state of an individual ongoing delta xDS session, i.e. a single type_url. +// There can be multiple DeltaSubscriptionStates active. They will always all be +// blissfully unaware of each other's existence, even when their messages are +// being multiplexed together by ADS. class DeltaSubscriptionState : public Logger::Loggable { public: - DeltaSubscriptionState(const std::string& type_url, const std::set& resource_names, - SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, + DeltaSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, SubscriptionStats& stats); - - void setInitFetchTimeout(Event::Dispatcher& dispatcher); - - void pause(); - void resume(); - bool paused() const { return paused_; } + Event::Dispatcher& dispatcher); // Update which resources we're interested in subscribing to. - void updateResourceInterest(const std::set& update_to_these_names); + void updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed); // Whether there was a change in our subscription interest we have yet to inform the server of. bool subscriptionUpdatePending() const; @@ -45,7 +37,14 @@ class DeltaSubscriptionState : public Logger::Loggable { void handleEstablishmentFailure(); - envoy::api::v2::DeltaDiscoveryRequest getNextRequest(); + // Returns the next gRPC request proto to be sent off to the server, based on this object's + // understanding of the current protocol state, and new resources that Envoy wants to request. + envoy::api::v2::DeltaDiscoveryRequest getNextRequestAckless(); + // The WithAck version first calls the Ackless version, then adds in the passed-in ack. + envoy::api::v2::DeltaDiscoveryRequest getNextRequestWithAck(const UpdateAck& ack); + + DeltaSubscriptionState(const DeltaSubscriptionState&) = delete; + DeltaSubscriptionState& operator=(const DeltaSubscriptionState&) = delete; private: void handleGoodResponse(const envoy::api::v2::DeltaDiscoveryResponse& message); @@ -86,12 +85,12 @@ class DeltaSubscriptionState : public Logger::Loggable { std::set resource_names_; const std::string type_url_; + // callbacks_ is expected to be a WatchMap. SubscriptionCallbacks& callbacks_; const LocalInfo::LocalInfo& local_info_; std::chrono::milliseconds init_fetch_timeout_; Event::TimerPtr init_fetch_timeout_timer_; - bool paused_{}; bool any_request_sent_yet_in_current_stream_{}; // Tracks changes in our subscription interest since the previous DeltaDiscoveryRequest we sent. @@ -99,8 +98,6 @@ class DeltaSubscriptionState : public Logger::Loggable { // Feel free to change to unordered if you can figure out how to make it work. std::set names_added_; std::set names_removed_; - - SubscriptionStats& stats_; }; } // namespace Config diff --git a/source/common/config/filesystem_subscription_impl.cc b/source/common/config/filesystem_subscription_impl.cc index 1bef8eafe8f6..80a693c69d9c 100644 --- a/source/common/config/filesystem_subscription_impl.cc +++ b/source/common/config/filesystem_subscription_impl.cc @@ -27,7 +27,7 @@ void FilesystemSubscriptionImpl::start(const std::set&) { refresh(); } -void FilesystemSubscriptionImpl::updateResources(const std::set&) { +void FilesystemSubscriptionImpl::updateResourceInterest(const std::set&) { // Bump stats for consistence behavior with other xDS. stats_.update_attempt_.inc(); } diff --git a/source/common/config/filesystem_subscription_impl.h b/source/common/config/filesystem_subscription_impl.h index 6ff089208dc8..851935f1638e 100644 --- a/source/common/config/filesystem_subscription_impl.h +++ b/source/common/config/filesystem_subscription_impl.h @@ -26,9 +26,9 @@ class FilesystemSubscriptionImpl : public Config::Subscription, // Config::Subscription // We report all discovered resources in the watched file, so the resource names arguments are - // unused, and updateResources is a no-op (other than updating a stat). + // unused, and updateResourceInterest is a no-op (other than updating a stat). void start(const std::set&) override; - void updateResources(const std::set&) override; + void updateResourceInterest(const std::set&) override; private: void refresh(); diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index c1eda78b6dd6..fdb3fa603758 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -164,8 +164,9 @@ void GrpcMuxImpl::onDiscoveryResponse( GrpcMuxCallbacks& callbacks = api_state_[type_url].watches_.front()->callbacks_; for (const auto& resource : message->resources()) { if (type_url != resource.type_url()) { - throw EnvoyException(fmt::format("{} does not match {} type URL in DiscoveryResponse {}", - resource.type_url(), type_url, message->DebugString())); + throw EnvoyException( + fmt::format("{} does not match the message-wide type URL {} in DiscoveryResponse {}", + resource.type_url(), type_url, message->DebugString())); } const std::string resource_name = callbacks.resourceName(resource); resources.emplace(resource_name, resource); @@ -232,10 +233,7 @@ void GrpcMuxImpl::queueDiscoveryRequest(const std::string& queue_item) { void GrpcMuxImpl::clearRequestQueue() { grpc_stream_.maybeUpdateQueueSizeStat(0); - // TODO(fredlas) when we have C++17: request_queue_ = {}; - while (!request_queue_.empty()) { - request_queue_.pop(); - } + request_queue_ = {}; } void GrpcMuxImpl::drainRequests() { diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index ae245c48d7fd..83821a91504a 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -34,10 +34,20 @@ class GrpcMuxImpl : public GrpcMux, void start() override; GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, GrpcMuxCallbacks& callbacks) override; + + // GrpcMux void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; + Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + void removeWatch(const std::string&, Watch*) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + void handleDiscoveryResponse(std::unique_ptr&& message); + void sendDiscoveryRequest(const std::string& type_url); // Config::GrpcStreamCallbacks @@ -52,6 +62,7 @@ class GrpcMuxImpl : public GrpcMux, } private: + void drainRequests(); void setRetryTimer(); struct GrpcMuxWatchImpl : public GrpcMuxWatch, RaiiListElement { @@ -100,7 +111,6 @@ class GrpcMuxImpl : public GrpcMux, // Request queue management logic. void queueDiscoveryRequest(const std::string& queue_item); void clearRequestQueue(); - void drainRequests(); GrpcStream grpc_stream_; const LocalInfo::LocalInfo& local_info_; @@ -116,7 +126,7 @@ class GrpcMuxImpl : public GrpcMux, std::queue request_queue_; }; -class NullGrpcMuxImpl : public GrpcMux { +class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks { public: void start() override {} GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, @@ -126,6 +136,19 @@ class NullGrpcMuxImpl : public GrpcMux { void pause(const std::string&) override {} void resume(const std::string&) override {} bool paused(const std::string&) const override { return false; } + + Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + void removeWatch(const std::string&, Watch*) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + + void onWriteable() override {} + void onStreamEstablished() override {} + void onEstablishmentFailure() override {} + void onDiscoveryResponse(std::unique_ptr&&) override {} }; } // namespace Config diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc index dffab9f0caea..06d1fd3b562f 100644 --- a/source/common/config/grpc_mux_subscription_impl.cc +++ b/source/common/config/grpc_mux_subscription_impl.cc @@ -9,7 +9,7 @@ namespace Envoy { namespace Config { -GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, +GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, SubscriptionCallbacks& callbacks, SubscriptionStats stats, absl::string_view type_url, @@ -28,19 +28,20 @@ void GrpcMuxSubscriptionImpl::start(const std::set& resources) { init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } - watch_ = grpc_mux_.subscribe(type_url_, resources, *this); + watch_ = grpc_mux_->subscribe(type_url_, resources, *this); // The attempt stat here is maintained for the purposes of having consistency between ADS and // gRPC/filesystem/REST Subscriptions. Since ADS is push based and muxed, the notion of an // "attempt" for a given xDS API combined by ADS is not really that meaningful. stats_.update_attempt_.inc(); } -void GrpcMuxSubscriptionImpl::updateResources(const std::set& update_to_these_names) { +void GrpcMuxSubscriptionImpl::updateResourceInterest( + const std::set& update_to_these_names) { // First destroy the watch, so that this subscribe doesn't send a request for both the // previously watched resources and the new ones (we may have lost interest in some of the // previously watched ones). watch_.reset(); - watch_ = grpc_mux_.subscribe(type_url_, update_to_these_names, *this); + watch_ = grpc_mux_->subscribe(type_url_, update_to_these_names, *this); stats_.update_attempt_.inc(); } diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index 9fb4cd76407c..299aa4f1480e 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -17,14 +17,14 @@ class GrpcMuxSubscriptionImpl : public Subscription, GrpcMuxCallbacks, Logger::Loggable { public: - GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionCallbacks& callbacks, + GrpcMuxSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, SubscriptionCallbacks& callbacks, SubscriptionStats stats, absl::string_view type_url, Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout); // Config::Subscription void start(const std::set& resource_names) override; - void updateResources(const std::set& update_to_these_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; // Config::GrpcMuxCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, @@ -36,7 +36,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, private: void disableInitFetchTimeoutTimer(); - GrpcMux& grpc_mux_; + GrpcMuxSharedPtr grpc_mux_; SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; const std::string type_url_; diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 9a182c2637ff..1be922742930 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -2,7 +2,7 @@ #include -#include "envoy/config/xds_grpc_context.h" +#include "envoy/config/grpc_mux.h" #include "envoy/grpc/async_client.h" #include "common/common/backoff_strategy.h" diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 94e920843102..8914519746f2 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -21,8 +21,9 @@ class GrpcSubscriptionImpl : public Config::Subscription { Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, std::chrono::milliseconds init_fetch_timeout, bool skip_subsequent_node) : callbacks_(callbacks), - grpc_mux_(local_info, std::move(async_client), dispatcher, service_method, random, scope, - rate_limit_settings, skip_subsequent_node), + grpc_mux_(std::make_shared(local_info, std::move(async_client), + dispatcher, service_method, random, scope, + rate_limit_settings, skip_subsequent_node)), grpc_mux_subscription_(grpc_mux_, callbacks_, stats, type_url, dispatcher, init_fetch_timeout) {} @@ -30,18 +31,18 @@ class GrpcSubscriptionImpl : public Config::Subscription { void start(const std::set& resource_names) override { // Subscribe first, so we get failure callbacks if grpc_mux_.start() fails. grpc_mux_subscription_.start(resource_names); - grpc_mux_.start(); + grpc_mux_->start(); } - void updateResources(const std::set& update_to_these_names) override { - grpc_mux_subscription_.updateResources(update_to_these_names); + void updateResourceInterest(const std::set& update_to_these_names) override { + grpc_mux_subscription_.updateResourceInterest(update_to_these_names); } - GrpcMuxImpl& grpcMux() { return grpc_mux_; } + std::shared_ptr grpcMux() { return grpc_mux_; } private: Config::SubscriptionCallbacks& callbacks_; - GrpcMuxImpl grpc_mux_; + std::shared_ptr grpc_mux_; GrpcMuxSubscriptionImpl grpc_mux_subscription_; }; diff --git a/source/common/config/http_subscription_impl.cc b/source/common/config/http_subscription_impl.cc index dda038436500..6e718e27775e 100644 --- a/source/common/config/http_subscription_impl.cc +++ b/source/common/config/http_subscription_impl.cc @@ -49,7 +49,8 @@ void HttpSubscriptionImpl::start(const std::set& resource_names) { initialize(); } -void HttpSubscriptionImpl::updateResources(const std::set& update_to_these_names) { +void HttpSubscriptionImpl::updateResourceInterest( + const std::set& update_to_these_names) { Protobuf::RepeatedPtrField resources_vector(update_to_these_names.begin(), update_to_these_names.end()); request_.mutable_resource_names()->Swap(&resources_vector); diff --git a/source/common/config/http_subscription_impl.h b/source/common/config/http_subscription_impl.h index 6ad8055e4e8b..727cc90e6db2 100644 --- a/source/common/config/http_subscription_impl.h +++ b/source/common/config/http_subscription_impl.h @@ -31,7 +31,7 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher, // Config::Subscription void start(const std::set& resource_names) override; - void updateResources(const std::set& update_to_these_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; // Http::RestApiFetcher void createRequest(Http::Message& request) override; diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc new file mode 100644 index 000000000000..d22593d0bf99 --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.cc @@ -0,0 +1,228 @@ +#include "common/config/new_grpc_mux_impl.h" + +#include "common/common/assert.h" +#include "common/common/backoff_strategy.h" +#include "common/common/token_bucket_impl.h" +#include "common/config/utility.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Config { + +NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, + Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info) + : dispatcher_(dispatcher), local_info_(local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + if (watch == nullptr) { + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } else { + updateWatch(type_url, watch, resources); + return watch; + } +} + +void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { + updateWatch(type_url, watch, {}); + auto entry = subscriptions_.find(type_url); + RELEASE_ASSERT(entry != subscriptions_.end(), + fmt::format("removeWatch() called for non-existent subscription {}.", type_url)); + entry->second->watch_map_.removeWatch(watch); +} + +void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } + +void NewGrpcMuxImpl::resume(const std::string& type_url) { + pausable_ack_queue_.resume(type_url); + trySendDiscoveryRequests(); +} + +bool NewGrpcMuxImpl::paused(const std::string& type_url) const { + return pausable_ack_queue_.paused(type_url); +} + +void NewGrpcMuxImpl::onDiscoveryResponse( + std::unique_ptr&& message) { + ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), + message->system_version_info()); + auto sub = subscriptions_.find(message->type_url()); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " + "subscription {}.", + message->system_version_info(), message->type_url()); + return; + } + kickOffAck(sub->second->sub_state_.handleResponse(*message)); +} + +void NewGrpcMuxImpl::onStreamEstablished() { + for (auto& sub : subscriptions_) { + sub.second->sub_state_.markStreamFresh(); + } + trySendDiscoveryRequests(); +} + +void NewGrpcMuxImpl::onEstablishmentFailure() { + // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately + // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a + // crash, the iteration needs to dance around a little: collect pointers to all + // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are + // now more SubscriptionStates. + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; + do { + for (auto& sub : subscriptions_) { + all_subscribed[sub.first] = &sub.second->sub_state_; + } + for (auto& sub : all_subscribed) { + if (already_called.insert(sub).second) { // insert succeeded ==> not already called + sub.second->handleEstablishmentFailure(); + } + } + } while (all_subscribed.size() != subscriptions_.size()); +} + +void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } + +void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { + pausable_ack_queue_.push(std::move(ack)); + trySendDiscoveryRequests(); +} + +// TODO(fredlas) to be removed from the GrpcMux interface very soon. +GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } + +Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + // We don't yet have a subscription for type_url! Make one! + addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } + + Watch* watch = entry->second->watch_map_.addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch, resources); + return watch; +} + +// Updates the list of resource names watched by the given watch. If an added name is new across +// the whole subscription, or if a removed name has no other watch interested in it, then the +// subscription will enqueue and attempt to send an appropriate discovery request. +void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { + ASSERT(watch != nullptr); + auto sub = subscriptions_.find(type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Watch of {} has no subscription to update.", type_url)); + auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); + sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + // Tell the server about our change in interest, if any. + if (sub->second->sub_state_.subscriptionUpdatePending()) { + trySendDiscoveryRequests(); + } +} + +void NewGrpcMuxImpl::addSubscription(const std::string& type_url, + std::chrono::milliseconds init_fetch_timeout) { + subscriptions_.emplace(type_url, std::make_unique(type_url, init_fetch_timeout, + dispatcher_, local_info_)); + subscription_ordering_.emplace_back(type_url); +} + +void NewGrpcMuxImpl::trySendDiscoveryRequests() { + while (true) { + // Do any of our subscriptions even want to send a request? + absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); + if (!maybe_request_type.has_value()) { + break; + } + // If so, which one (by type_url)? + std::string next_request_type_url = maybe_request_type.value(); + // If we don't have a subscription object for this request's type_url, drop the request. + auto sub = subscriptions_.find(next_request_type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Tried to send discovery request for non-existent subscription {}.", + next_request_type_url)); + + // Try again later if paused/rate limited/stream down. + if (!canSendDiscoveryRequest(next_request_type_url)) { + break; + } + // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. + if (!pausable_ack_queue_.empty()) { + // Because ACKs take precedence over plain requests, if there is anything in the queue, it's + // safe to assume it's of the type_url that we're wanting to send. + grpc_stream_.sendMessage( + sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); + pausable_ack_queue_.pop(); + } else { + grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); + } + } + grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); +} + +// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check +// whether we *want* to send a DeltaDiscoveryRequest). +bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { + RELEASE_ASSERT( + !pausable_ack_queue_.paused(type_url), + fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " + "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", + type_url)); + + if (!grpc_stream_.grpcStreamAvailable()) { + ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); + return false; + } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { + ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); + return false; + } + return true; +} + +// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or +// a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). +// Returns the type_url we should send the DeltaDiscoveryRequest for (if any). +// First, prioritizes ACKs over non-ACK subscription interest updates. +// Then, prioritizes non-ACK updates in the order the various types +// of subscriptions were activated. +absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { + // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose + // type_url from pausable_ack_queue_ if possible, before looking at pending updates. + if (!pausable_ack_queue_.empty()) { + return pausable_ack_queue_.front().type_url_; + } + // If we're looking to send multiple non-ACK requests, send them in the order that their + // subscriptions were initiated. + for (const auto& sub_type : subscription_ordering_) { + auto sub = subscriptions_.find(sub_type); + if (sub != subscriptions_.end() && sub->second->sub_state_.subscriptionUpdatePending() && + !pausable_ack_queue_.paused(sub_type)) { + return sub->first; + } + } + return absl::nullopt; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h new file mode 100644 index 000000000000..1bd3b5a1f174 --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.h @@ -0,0 +1,117 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/token_bucket.h" +#include "envoy/config/grpc_mux.h" +#include "envoy/config/subscription.h" + +#include "common/common/logger.h" +#include "common/config/delta_subscription_state.h" +#include "common/config/grpc_stream.h" +#include "common/config/pausable_ack_queue.h" +#include "common/config/watch_map.h" +#include "common/grpc/common.h" + +namespace Envoy { +namespace Config { + +// Manages subscriptions to one or more type of resource. The logical protocol +// state of those subscription(s) is handled by DeltaSubscriptionState. +// This class owns the GrpcStream used to talk to the server, maintains queuing +// logic to properly order the subscription(s)' various messages, and allows +// starting/stopping/pausing of the subscriptions. +class NewGrpcMuxImpl : public GrpcMux, + public GrpcStreamCallbacks, + Logger::Loggable { +public: + NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info); + + Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + void removeWatch(const std::string& type_url, Watch* watch) override; + + void pause(const std::string& type_url) override; + void resume(const std::string& type_url) override; + bool paused(const std::string& type_url) const override; + void + onDiscoveryResponse(std::unique_ptr&& message) override; + + void onStreamEstablished() override; + + void onEstablishmentFailure() override; + + void onWriteable() override; + + void kickOffAck(UpdateAck ack); + + // TODO(fredlas) remove these two from the GrpcMux interface. + GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override; + void start() override; + +private: + Watch* addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); + + // Updates the list of resource names watched by the given watch. If an added name is new across + // the whole subscription, or if a removed name has no other watch interested in it, then the + // subscription will enqueue and attempt to send an appropriate discovery request. + void updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources); + + void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); + + void trySendDiscoveryRequests(); + + // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check + // whether we *want* to send a DeltaDiscoveryRequest). + bool canSendDiscoveryRequest(const std::string& type_url); + + // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or + // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). + // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). + // First, prioritizes ACKs over non-ACK subscription interest updates. + // Then, prioritizes non-ACK updates in the order the various types + // of subscriptions were activated. + absl::optional whoWantsToSendDiscoveryRequest(); + + Event::Dispatcher& dispatcher_; + const LocalInfo::LocalInfo& local_info_; + + // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All + // of our different resource types' ACKs are mixed together in this queue. See class for + // description of how it interacts with pause() and resume(). + PausableAckQueue pausable_ack_queue_; + + struct SubscriptionStuff { + SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) + : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), + init_fetch_timeout_(init_fetch_timeout) {} + + WatchMap watch_map_; + DeltaSubscriptionState sub_state_; + const std::chrono::milliseconds init_fetch_timeout_; + + SubscriptionStuff(const SubscriptionStuff&) = delete; + SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; + }; + // Map key is type_url. + absl::flat_hash_map> subscriptions_; + + // Determines the order of initial discovery requests. (Assumes that subscriptions are added in + // the order of Envoy's dependency ordering). + std::list subscription_ordering_; + + GrpcStream + grpc_stream_; +}; + +using NewGrpcMuxImplSharedPtr = std::shared_ptr; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/pausable_ack_queue.cc b/source/common/config/pausable_ack_queue.cc new file mode 100644 index 000000000000..0217717de035 --- /dev/null +++ b/source/common/config/pausable_ack_queue.cc @@ -0,0 +1,66 @@ +#include "common/config/pausable_ack_queue.h" + +#include + +#include "common/common/assert.h" + +namespace Envoy { +namespace Config { + +void PausableAckQueue::push(UpdateAck x) { storage_.push_back(std::move(x)); } + +size_t PausableAckQueue::size() const { return storage_.size(); } + +bool PausableAckQueue::empty() { + for (const auto& entry : storage_) { + if (!paused_[entry.type_url_]) { + return false; + } + } + return true; +} + +const UpdateAck& PausableAckQueue::front() { + for (const auto& entry : storage_) { + if (!paused_[entry.type_url_]) { + return entry; + } + } + RELEASE_ASSERT(false, "front() on an empty queue is undefined behavior!"); + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void PausableAckQueue::pop() { + for (auto it = storage_.begin(); it != storage_.end(); ++it) { + if (!paused_[it->type_url_]) { + storage_.erase(it); + return; + } + } + RELEASE_ASSERT(false, "pop() on an empty queue is undefined behavior!"); + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void PausableAckQueue::pause(const std::string& type_url) { + // It's ok to pause a subscription that doesn't exist yet. + auto& pause_entry = paused_[type_url]; + ASSERT(!pause_entry); + pause_entry = true; +} + +void PausableAckQueue::resume(const std::string& type_url) { + auto& pause_entry = paused_[type_url]; + ASSERT(pause_entry); + pause_entry = false; +} + +bool PausableAckQueue::paused(const std::string& type_url) const { + auto entry = paused_.find(type_url); + if (entry == paused_.end()) { + return false; + } + return entry->second; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/pausable_ack_queue.h b/source/common/config/pausable_ack_queue.h new file mode 100644 index 000000000000..46222a8b2e3c --- /dev/null +++ b/source/common/config/pausable_ack_queue.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include "envoy/api/v2/discovery.pb.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Config { + +struct UpdateAck { + UpdateAck(absl::string_view nonce, absl::string_view type_url) + : nonce_(nonce), type_url_(type_url) {} + std::string nonce_; + std::string type_url_; + ::google::rpc::Status error_detail_; +}; + +// There is a head-of-line blocking issue resulting from the intersection of 1) ADS's need for +// subscription request ordering and 2) the ability to "pause" one of the resource types within ADS. +// We need a queue that understands ADS's resource type pausing. Specifically, we need front()/pop() +// to choose the first element whose type_url isn't paused. +class PausableAckQueue { +public: + void push(UpdateAck x); + size_t size() const; + bool empty(); + const UpdateAck& front(); + void pop(); + void pause(const std::string& type_url); + void resume(const std::string& type_url); + bool paused(const std::string& type_url) const; + +private: + // It's ok for non-existent subs to be paused/resumed. The cleanest way to support that is to give + // the pause state its own map. (Map key is type_url.) + absl::flat_hash_map paused_; + std::list storage_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index b85c8bbb1d1c..8e8fbc4e088f 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -5,6 +5,7 @@ #include "common/config/grpc_mux_subscription_impl.h" #include "common/config/grpc_subscription_impl.h" #include "common/config/http_subscription_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/type_to_endpoint.h" #include "common/config/utility.h" #include "common/protobuf/protobuf.h" @@ -21,7 +22,7 @@ SubscriptionFactoryImpl::SubscriptionFactoryImpl( SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( const envoy::api::v2::core::ConfigSource& config, absl::string_view type_url, - Stats::Scope& scope, SubscriptionCallbacks& callbacks) { + Stats::Scope& scope, SubscriptionCallbacks& callbacks, bool is_delta) { Config::Utility::checkLocalInfo(type_url, local_info_); std::unique_ptr result; SubscriptionStats stats = Utility::generateStats(scope); @@ -62,13 +63,13 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: { Utility::checkApiConfigSourceSubscriptionBackingCluster(cm_.clusters(), api_config_source); result = std::make_unique( - local_info_, - Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), - api_config_source, scope) - ->create(), - dispatcher_, deltaGrpcMethod(type_url), type_url, random_, scope, - Utility::parseRateLimitSettings(api_config_source), callbacks, stats, - Utility::configSourceInitialFetchTimeout(config)); + std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), + api_config_source, scope) + ->create(), + dispatcher_, deltaGrpcMethod(type_url), random_, scope, + Utility::parseRateLimitSettings(api_config_source), local_info_), + type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), false); break; } default: @@ -77,9 +78,15 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( break; } case envoy::api::v2::core::ConfigSource::kAds: { - result = std::make_unique( - cm_.adsMux(), callbacks, stats, type_url, dispatcher_, - Utility::configSourceInitialFetchTimeout(config)); + if (is_delta) { + result = std::make_unique( + cm_.adsMux(), type_url, callbacks, stats, + Utility::configSourceInitialFetchTimeout(config), true); + } else { + result = std::make_unique( + cm_.adsMux(), callbacks, stats, type_url, dispatcher_, + Utility::configSourceInitialFetchTimeout(config)); + } break; } default: diff --git a/source/common/config/subscription_factory_impl.h b/source/common/config/subscription_factory_impl.h index 3a7f0ba1e31c..43cec2139649 100644 --- a/source/common/config/subscription_factory_impl.h +++ b/source/common/config/subscription_factory_impl.h @@ -16,10 +16,12 @@ class SubscriptionFactoryImpl : public SubscriptionFactory { Upstream::ClusterManager& cm, Runtime::RandomGenerator& random, ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api); + // TODO(fredlas) remove is_delta once delta and SotW are unified // Config::SubscriptionFactory SubscriptionPtr subscriptionFromConfigSource(const envoy::api::v2::core::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, - SubscriptionCallbacks& callbacks) override; + SubscriptionCallbacks& callbacks, + bool is_delta) override; private: const LocalInfo::LocalInfo& local_info_; diff --git a/source/common/config/xDS_code_diagram.png b/source/common/config/xDS_code_diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..ef4df79cc1e30f59f865d5c29abc91ca05b19f72 GIT binary patch literal 97291 zcmY&=Wk6JIwDru;ozkszmxLfgcPNN-NDb00Js^!B-7%Daga|`7iZl#JND0!?-SHjY zd+(3$-*9Hm%sJ=T&)#dVwKmZ@TFL~tG`JuTh(J|E;Ux%!)&&AV=&&Kckp>wZ9uNo) zQdN-EdpEb+PUQM(CU;YHBtdZA7VGVB?OUSNlJ+L0>fWj2l+S6OOMXmd@#;FT6}bsJ z*VD59xW&N@<4}rAl@%#YwC7EX4pjsxQ@O4gFmFc@V>~gAk4cmLq?i!BOM%?|5hphU zChN5@613>N%ee@oqu=-U^SeID%PT&qc>QJn*~RTuuE$vGg-6hZ&uAv`@k(zjCBIqQ zyQ7s}^&X*av5^1!3pmVxX#aQoe?I`)`@dJfAuQPco)nut&i(pJ#Q9$Ak^S3ar(~XH zH>-EYl{zY(v&VN<~w1HLi1t`;X7BS2uBf0?}FK6^6o_h)H$d7a&uUoD-C`xdIeF3)%4`fTU^ zX|^BO6Pj=TQ%=|}>-|770XA;7g5tTbk2Csb(BmLm+pPX%w#-!tkKv=q^2Cf<#%}@C z)QQ7a8;&AKLwH2{0?{%Dv0gJ7yOPi*=nN`9xc)d$C$%mobPv&}80)M{D~-b*KUuT< z%qrq)_BVqbr`Scq+UzU|B;pNcPlT>HVpiXBn#V(5(N&G0@P{m;!=0jbef0Cg5_Ag! zYv*~L%w(O+lW_rf)`U<@CDOtf!IT~Lm7q}FQS%-g(u215Y2%d`lRWMU+*|kkDF*5- zRP8mM2b5O|>Q7xd@$=`&ggQU3dB^R|(!>2%t3F9d$y;u(2Q_JGh+LIPb8SgUuX*AT z+tSsj50~kDR=OTe0v1(BG4^YVv+XF}y5c9HA9Hgzc0(~X*gx2Sjid<3VjH1Q>scWW z69ZU0M(vyLb4DIg!9n>T`b+FybQFugI})z?OAfV)vZ#!F-qM#CBYyC6{MRiq_-bY<^X*I7DH;sNjcboBaz&;|Q%yAyBx?u#izbY6HyCjkOj5C7x{D|Av$b4n zBJ*tFQ+ri8+GAP4=Q^U}rwmMN(Bvu_odi>T+v^Vz*1-P%@a*sYe$YmyUB^ZHftQ!p zRbN|hNuFN$moE?Jjh%b^-Qp=Hn`6QXodH+1o$H?rpw>}X=a58GO2YBaUc&H9bOQ=R z)w8e+4Jg~%6ghPELx=!%s?C#qJcR02kaeA&b1bDY3*{rlH-oM6#2>Nu3$(5O^vn-e z$SJ+Gs{qXzIaDB)9z7}m!=*T$^yml?A!((mxZod1MG3zVHM;Ez@>)nsI^qr0_2`q4 zl)hB9aV9aTryy`}gnxc(mN4@))L#*@eYP%8bVO?P0ou#zK4aJ{#(FN_jWRo<8k`FWo z{Tei9JD;t4Y(@votLy|PVk}ZvDthNZ=_#}W54qUa+(m|+KAYaNu%>3 zkfQH?2Dahw%%>b@9;&DD9Pvy3sA@Z#Hl}^*c<8f8uI}GjuvYOp?#TUU#f-KJ^x`>b zf$?)H$y6q2tJV)OXR2y<_B8@}1V(+?no&SW_8D(O9uDcOpJEt3ZK0yCkyVYYYUnB! z(Ye0))K?A(CisR`BypXsVg!dfcA`~8#;cAYC#NHjQKd`VmO_K=Y7kh`!a~V=uVv&r zVU#WjUkjb$T2Dy-jwB(JERK7${u4G$oRa?ge>bsT+|+crDf7@AjQl9?3*_)dr}?y% z`*XYCbGwf9zABR@SIfT>CHK|YYG13{g}i%m(+by+0=~5&T3c$mOW%J2k4aYcyDDWh z7te=xdKVa|CvGy3l}k`nd04I6TnTenMwupI8LvwHJ55pt>M8Orn17o|vOTV`!!i0> zQHK5z(MK~|a#PeC(6LObE}SJ%v%K&{1*F2wd3;%^mG@U9rkL3Jt{- zg^81)?5NGbwOu8ySI{Ze%xRdXfeCR%72}u zz~SK-09UdW@-a=e(Wn=dbD|nu^w+Nb=4y?USa=$hmaXHyLlk%HPV<$H$B3Q-I{#D_ zOa1G}2+8yeRz!SeGajm-rmumX2vK({wt|K$h}>>!Fe8-(5_{ z!`gFcjazy+p?bg`>Uq?9wYG{ zzSXX;xg67rPVtNVPEKrgAi)j733M*s<;QEdYoGF~Z$c=#qr6pWL}Sdt3>@5k&AJDQ zV)IZ5OK2)IwKC*?wy^qPPd8LQ_@ssq^NM4^w2mdAd3xQozOcYEWMN@J4o%`P`7;I| z>^4T1BTki+aB^8qeVopLZiS)s>br^492ye1G&8A@%lnO9;AxLDV-eBJU{EVtg__pd zQ2BuArdH;={y~;J+koJc6`=ZU5$`;+Y!?F3|35&07WeP`fb^4<679d+LO~dUrj%{( z{{MIW2l4+0>{8k z{z(5U&5u7BHEJ00Gyq2H1SPGU2AI%hImyLMtxkY^Bv->MCbVo&2Tk1%GMQ|+lS>NG% zHqG`V;H&G=y@{eZ4-H?uIQRq)jP_Q?3HYQXdb1bDH~yRDX#NR#-dvMo=B%2)?8Ksz zH#)NQ57YX(l00#~ZZcc%FC$>NDES|wp#Y2+aI@Fw`=m7Ks*4hLZ+G|f@)-lj@8*23 zXBys4eT;@UXumL%v=lPh7*}Qt&m&@dzKh9c-a%(Wiah(2qFN1|imZwUT6({< z2KK{8^E`bX>Q*?CODiaoo|-BBfZRFDK;Kfxa;Ex{==6}sP-oug9e^Kx;fZ_}@-GZ* ztU)x^vkgx3?}?s^|4Gxxe=g0>&)=VlXfn+SoB#+<25GTWM^6O7?bAw)p%P! zV^)N?ySZR@-bH`&vU05q1Urnc_L?#E>D@KbVOm7g)=m`uX|91*vuo!)S<^5OUuF++ zX**If@1V$8|J)oHT>-Vj`;IR1FBT#h{8VJd2hGH3Lx?lWc%dg`#G7`WyyA+@6TB_# zWQBDwDk|u)=9FU|AM~fMXTa_+K^6lSg7QcyzS-{`8RB0Z&&Yqv8+*fArOlkGIMaz}S}2^44I74wOQO0W!dtyF zbz$p9wXY0Y?HS8=wf$p&Qc5xw`9n!}ed`v7o#tA{ZRhbH2+}BNfr~qi#=5K}L4bQm zVQAp5lk_@U9dbnT8b!_lbr+?aeaVq%)quuGSX|26-ylPMoNSw@_c}YjI?yi+eElPi zc71()LE*fYbpK6NRh6bU_o+zQ_V#x5B)&#rOe@L$^S9qFRV7J1MURs-;FvnKb#=a& zOa2G#0KWa*5qJZ9!I>KUG|u670o;CPyzrdklM}{w%V0SM|J|?OrNL48QUq9bewF3r znc{9Ns{gZ|n9N8eSy|cPx&>mB?MIYGGWHy<`wPt;8^gXU(d@ejJT?mB`GR;l;U}kj z4(@DhmAezAyps&vb{9O9xx}WLUTAtPH~X!rGSt=PoxhbZ?@Bc?SK{SN-}}6J^@zE8 z10U(wfREg7JJBNSms;)tU(4KIu6}%eU8Xs-q`S<0R*4#c1g65S`TO+VIU8K=j9Xc4 z?}8;YQND8`@8M}QXuCUa^0jPi&s$!)@_3@jvulYkul?{*QgY!UIV@I9)Ky3}t{bu5 z7^af=l~V+6DP&pcWe#aheOMc47i7i&uY8~;vFX>@Rng=e_rm^~Sy9_nqA-4{TWUM@ zbg9lZGohzvUWmCiEd^8Zm8Qs=#dzU6-srhGToP%2Kj)LziGl{skDfMOojk^`S}10= z$H7OwrxU(}$7CIAWnIv%t*!MmdHuT4 z{^zIC1?Qw{A>f%R&G)`EFmW^Jd0+Yi$4%^zd7y*54RQop%$GXuf9!`rCy=o>TNmld zFt6%94-&PYpLr4idCAtT{4QozMAj{FsMQr6{aT~VZS4`kAHgy*4-fq9kltG9_r(eb z%JLmZtS1+7g5ZY_-)p@qK1MI9YNgvAQfN-#KtyV-2S{f>W7N0$9Ha?S*&trQ83Oer zSr(c6V{D_!NFam6h7@i#u^T$a4K)gABKFiO*iBbfJ>4RT8hA)Nln5kFURH!@)jCPt za;Nsqd|gXR)>`p|HWBJC6O*1{CYV3!v4z7nB?A1jnh(tY^+;y@JC9m&cWpUmXpb7u zVY+oCkZ4Hgh`8qDyTo7Z?~3HGw~-k}VWej35OIW^Tt+>vsv=(|lTDIIEo2IMDi&#E z#1hZgGg8QsTw2-2q9`I8d47)lesHtN9KMmAGmx)D_Pq$CxVAy#j^SP4v8`pwffu-J zS~$>8znD~}EL_<910P8%19pyASJxC*&}v8t%}XiNef6rn*$oF38TR=LQ=Z@736-fW zZ`8W3^V?u_dep)rqf~6p47Fb}P>F4D0?&! zzw6K{w*@7e!k^im9UmE_2jA=8%0~Tuj)so@1Km97q<@o#$bM+ceZi#N&kOmys~bqH zm9pc3v%CjUuvNZ(x%(8-JGEThLjpV1N5BiDH`=%xg925%BfbK89F^6S@}DuxJXz zA{*oq+moHfd9h{6FAy{fmMm3Y2)tN4Ux7?l4-q@wttD&Et&pO_b`ARoE}UU4_)Nx|?w4>rqb{Rz}$A&0g`cu!4w3(R26VZfTRTe-&TQEB?A`fs%x z(a?^<$}0Tf42GT&hs!Ds&UD1k`NrE?bbb@6ulmPd;g7{^vT^i$4iSR0qghisK0DXY z#s;O`X{qh2{^z`8m4$9GQZ|+0%T*h2HfB5u1=4CWdP_gOBxZ{1;!lXG;cu?R%fFqW z9Ax}%q+r+%jqKw?GUor=mv zNB(Hx8jxk==9>CCxZA3#4pf`AMq%pzZgMNCs^Vp5$8ue>lb4rALZt3JJUpV8H%!dT zMzt}H4cfhU<0Nv3)f6^qU+k0<=T9`t*xK5@JfCh6j^xE4(o8FNmfv1Iw9}=xhYho=APN3m1f9pa;{q9-h zkAJVzRmBQIfe)ONQl}H>un;B2P_mb^ol?9Jqw)b2t6)E>@KQ!?MPiEKTE5ZuAejH> zYCcG;)E)*(pUH@}uB=~xn@U+D#7hMFP1ap6A=8i%#JAit{+pruP1h-fs3OO#W?9`D zC;DV$7%%USOB$W2@UtNBTwYSX60%#D$zgB(x<8vmOZ^-JHNl&Zjr)p}Pd3Lp!9!T8 z%(f9P19myi53Ptof?!nph@F{5EGQX-(ET+;<4)h8ED@a3H6?r)X)@Fx%|FoRont5= zKrXo0+slg8F52{o6GG9tpFAPkenq@be<-+q%)4GRvbUw$ob%Sj#T=k8*9MXcJhi#W_wo3?oV&G)62KVIh#zRF&@-7vqKGQaC)yam(oeDl!MByc@B{6=J4TWSoT zPm^VCEz;U3W&}6N`{DjB>+BH-R`O~zTg;_l+Vh_0y4h}wA(PgW$eHnOIjeVeEwMkDtC{~(#@|d$ZcgG@G8h9i(^%?q%QQEx%NOC!t zw5h}U@SB&lfa;ezwR10eD|cTI0*wr95{KHM7Yjz6Vbwx4iycC70+L?Y!Y4g@HZn-V z&+3lp`gN?QqOvoVGW*tM16g-%Z9iSx$0AJqJmfD3$}Ofi)W214@NJNf#z?Pz^yeF8 z!_e_RX$1l++>k(cbik#dDLu3OtaJGC(_rdoFEs|rTHTL?ty)Fw?neucMBMLW`RuJzP+3PihywEG> zJ2WId&K*Q#Oe(gGX6akFPNisWs7=g?o`Lh(>V#w8Y<8bh*Q2n=EquFi$~-{}4g-6M zn~4&2E-jRXqBC*-k3T5Y(5yDD~h5XAx@WUNg#$9~a<|rc>`;1|!p5y1q z0f62d>{16LYP`S0170Sp2dBRY=9rkjELV4JUWXN&m6GKb(tx(0Mlm%g zAPKA=(H)v3nMeK*U_ks-AY>4E@}x9WLE)($ys&`zjU9mnqxOrM`dXo$bRqoEk96PI z1oHbe%Ga{NkGse@n3U~gvjf;wdO}#ee#K`9Sq%MtBpppG^(f_J4eLOJ=wq#p!BZ(lGD?K&!8v;D1DQ8@c4Mr9k zrk2?bR|tHNi>}bGdSkF6>oGiZb+Y+z{pI0$$o&4;r}M^iM*8~KS1Y&aP-6L}nnypw zziF~)o3vAWL_MzGwPdEwD6s0DDrh#X? z)A3f(L{oYKW=00_Ya=Y7ASof zFg>u0qX=s-x#;p*W9%3X;Pke8|ACnOZYFy%e%MJSI4%bu`7{6V<$&=YceSTIa_<%qG+b6FC;NJ*%BC>~3$<EL|yTJEEYzq5Qa6?(Wa*6p4bIS zEd+2xxdRf(FIZS|f!8lr&Ya6-dk`^4ylZ*!OLFk~-q(4ZHRyfFrzq-bx=FUd*#>PU z52P2ZY%>Op)Tiffu_iF*Liz>gyLnZdx2EHu)k4$RHeukomQ6x(L^Fq}N zdz4Ru`Matv{JuR(NHs=8=&HYSA!&UenH^)xU4FCD)}uw(P=d*WB|wm8&QS`4DxP{O zufW7MXdGLwZ?cg+F&?%Ca6otp|Equ)%$Q(TjD#xm~iX`>2135A@U z-DL|3R5(&qoUtMS#*8@E7sYBt{W--cY12kQ;efj`UqeZWLv*9>9!zt|UGIE_28LZp z%cnIrH-ol5Pz!(W=7SC$_|75i0V#EKa4_^SKgJJ-Kf@tLEY`NRqxmw76N-$rtmuo_ zA)-4y&R5YO5wSiDOIgNd51qmow&#Y5r9j^)YMSJF`a&NJx7CC9tnETXQVW$t2wmG; z|HzTF0>my6JzP|3#pyyvzhW`GivW7BeC?${|hKHAJZAXJVnRsDx z5kgKsz_258i@y%=A95OV;8U5Yh@hwZdJ>N(^(n5A&)!6Km|`}1N&UjQ6nY2ZwnQm2 z28t{{Fx|LyO?;Ua_ui!<7!_OsKcF{@CD7V+YdWZVrO{J;2S) zZK)X)BkrXUR%P7yoIDMP46JIlQWX_<;(R{Jz%lep%9dW>X4={oMC1C_Ib1Dc3UcdEHEV&S|7{n*~wnGJ&Rf^$YT)O{cafBp?G{{Y&jv*|gB$+@}7 zf$n&gesI+51_k|9VqL79sR_ngJ<v#c*ZuhHOf?b8D5bE!kcH+|}gHo7T<~ z4vFXMXEWxTs$U59E;&yM@B~qHx7~Y+&6o5B;0MJe?W{8fEQ3n)`Q=Fj zaPd=yln0*_@cgl>eBtsta^mt*~*HokG8j;mn(Z zPR7H)%RiBp=2cd_4c>|AB=l>VDq32j=^dvff}N1%3HFDz$esJ9n{d_AhTEGFU!sqD zgU|0yfBqxoTD%+r1Enbe0v~J3T17eXEACi^B}(E|a$ws#Y8<$;y}cnU9qUv`s;Fe6 z*_-tTNvdAfU|VrEfZ6^4W!rr@{2t(I8G>)!Kz?`EXXo{21@e(h8vv6|FB23nk|mn` z)ai3Yg@cO=I1lGsMz{X~lPPX6WP!dJHmRw{i#=vU{Ca4c`JMV$>`U7!L)P|G8b^r#J#ei=}we9g?vx@J8!2Kx@$kN%EzUS4&y`o(WN z?B|bt80&bUtNU$TOFJTnm*-CD=X??nxEq_BGQj-?$(N8M_AwZDUj*N{_&yxTFyx7p z--KX&epu6vXn8RM;9D`rU#`ntA%HM{v)S~YqyhC;HHSxJ|0l`E?_oC2>H8Mh?%Sna zm^}Mha`R?W{D2{nkRBq9HXr|i;;V}~V20?soliohzBX^ajtst>ecf?RjxZ0IIX zS^2`Xoy&Kps*#I}D~;dW%+S#H?%L}?)B`$G&-d)q~^1=}sg= zv-?KE0qHorHsIawDrfUnG%vrpVuu!*ln1ne$(z13|Bjr@jPD2_w^y zXj`|px2nl(yntt&=b#w!ba-kR$Lv5S-f zpWw&d2j9AqroOSP$@m^yXSr=TZGq?12SET^vUPd1YE`-K9lv~F?(d#6OhT6v@Nlyf ze6^mg9NS=Z?=%1_*yIoB<{5kzc)j_szZ7`y^02o#aE|sS;Q?@|0OpvV?{m{V!CMm( z696CDjQ0es7SL-xG+Voe0`>$n=^4qJh0V3K?@K@ZslbvQzxG}vJ(>G7-|G5~+7vWm zG2t8`hPgKA|3z^F`{|<(^Ae%aL&h%cXCp2_9!S`;h^4d-EPfp)qhk->f-iSt&2K$V zHiFB`o89+Kd~gSWl5TbSW5BHK0~OTH+FD^=4QKo*>($j2z_%M38vg6Sot#ESMtC_n z2Zo1pgze(jSAp$IK|w(ZYh2#n-^XOB4@7v8lvh_1hWjWolhKZT&EWvSCPs6gr3c{R z;_i;l52f*Di#kw9xXd9{f#lmeLKq8)R(k#cJqq}R=+U{ok7!)A7TpjbcE;vJn8Pg4 zlE2_%<@(ehQ=GL{Lfwn48nKche>X;CGy+dzhpkXe35tKG}#KcsDPZrY1hpc zLOw%7@s1Mp1Y#4C2f(H>OH*-is48Y9=%`RJHVVr0Xk2Oi5b5;?1%gk!cy;5Y{>lke zl}pa9m7@~a?X%aWSwODb{5rRuZ-8u6|K38{zJ$xj4bj_Bj$zo1LVn0j6U1y59A>g( z%3cNK{*$%9E&PHW=-STjvt8~lyS3-eZzMuhx7q(PS?xMSr+;4ug5CUPZbs^tuPPp# zf^6OpK*zd`=#u2`L;ao#LG2s+F|8$7t?JG$VED1|A&$r3fM@~#bH{OSI(S3fz8*1_ z4nK->^Y7@W)?lbOMs3UhxRUb={*Kii1nK1#Q4lT9R*e^l0gI_ScaK>v<&x7dHvf#6 zP2lKalHJ8K#yV$z&E*i$$SXhoGD+i;W;R72@4F`kMbE=OH<6%k0g#8`dB8hmgC>!3 zL!N2gl&|PeZwOrc{3EiO(S@oSD{Nrcvayp{-E1tZ#D zSmbna9a(I$@`7mY@sy^1dk;J333MN68UvhGC02Zn%%MCP_)JT=? zGFhY$FJY{s147@?gBzkbkMKcT@o^hoDu_VmS4^r@sC1?_8XRW$Y9U3Nqq(#rO%#YC zVHQUu8x8yPXlP}cXNLMK6~0QqJBZ(zV1 z_f29}+}zk0jCY`HTV>Co!9v(5r>6_M85tWF0S*nWH2tZmDZn7+USu*6np$1ah==+d zZrZ7hn-HVX&vh*qH58&!AVnbl5_I>=>SMt+8j)0u|HXbXyLQLj`8=>3ez!+`IOufA z$W2I!s?~2mASE9Z4BubsWTfZtoNsXYi;42}^%YSJO)O8L&0GbXHGXi07!%)v`knt4 z>z~)~p9uRMfY9ymPog`xDe1ib+B#6o0z1NZCF-zCb9wCAtKT70N?lFOF|W6`S7Fso zSNA8t`eF20v-((qk!O#>O_(i0#F=ya+VDoK(+M{Xb^=RTo*HL&8ei0n=e zxawuuo52SvrH?o$i+`+gD&4@~VC@G65bV{Umh=-|oPVZC$W|)%>y5RwU(D!|AF!bZ zzrLc~F$SKF!*o8?OJFRstu~Ym|EEK{e+4>8cc2$2rG*lyKtQ>9NI@e+u}}Q+lGB+7 z`ajzuU=J9I0wbwtn$Qp44Uc6AqIuRi=%a(O#DUhAsdl$}#VRVf!r%_JUI$Lpsao{u--%D7kabui~h;_pu^0kECQ(vvA{SJrek zxJ*uhDn=mSa*Y1Fg7-SrejmUVFv9+PD4`>I7-3t`-L*1b?w zKw#)St=@yVm6PsQ7_Syx<2f<1*5J3V=&0SH0X>WLoLw01Y$%#QQJDh^gB0GSeVPe< zJ7yPkbv%it0UOf|T8+l~$7H8IVhY9Njak+X+HM-7hi+Z8#^ok9WwCnoRcdU3PQg(^ zIy57!>Q_2xD-3kd_jv*w*cmF;(321>h9*EMUjOo81I2*#fqPjokwYfrMYc7-ECX1e zZKt}G17fCUEr{eGc2FkNc95B}uh3CN6J`VvK=2>}n0ZXQ~ z@yjxF=ODoq6FITpfQE|wd`b#>6TjW7P6PI2A+72E)QQeU60c2-`eqAITTDiwC+Fyi zHB9)|qt6bYUXbV5V@9fYxVYFM*;O)_iFXGLo!DG4UsW&KAO9 z>>yS{IAcVJS!p`|H1T40wkQ$J8vm7XUIbi%JB_tqN|y@*G)Irar2YJ=xWcY0{2N5% zm6F(8*o0TiaI{9$ZxDxF)KUkr4jF3)Nadn;l)(HdScwywEfGqfK1|mg!Aj@vse;7^ zhyrZEaRB!DuZRKwc}t-LXvx2`WWZ32?%rKwy5i<+DhQYBue(gP$GsF-H3B$YuF48wiYRh3g>RQUp_HlqN_88RNSQ z=}q~#0#)thB^3qqUynbs(VHNaB=y2ULsY^%+YHTNYx5dWqIprP$_GkSDMj4(xe1 zyQwnQZEs$}F2c06_EZjig|Rs#ymvDKOd-dBG)w4Sp7xOsDDpC+@xsapd0iP?IxO!+ zZsrQi|27gtz97w?Qi~mz**oSb*K>!O)Y|`KlPvzv2=!lu0bf6Lcmo(TEs^0W{&ZNZ zP9mCQA8=48wC{8$DK7_{bFrb6mt zjw+n*68IGSI{9M@(}uzZ_FF2Y`?nN+T7ameIf5Du#7STS(Ps$IC_|((^S!6-uU!X# zF6woakr&|6eu0ZJqDxjEyL)0Y9cauU{bx8cgtz(E__!d?7tiZd0Da-oEZoO)>Q4NR z8*bv-(gUbS@`{0rTV&uZ_^OcW<3`+ORfAyjojNe9j)_H!CCF#`i#cE$QoQAZe(Q5~ zXD|ixW51xRsSmhvH|AI}GM7rw*Pd120jO1h>u*FdbO0S)MS_Z(uK5NJfruR6%;e#> zUt)l+>llZ(I(}tJ!AD0&BMkQflwSkfz_)Vr{PL{FRI|#5?K7l*^YQ&5Atoc=o!ZOA z_6Z?~pFeA`eB67L7o|0&73%y=3y9dgqI#V63L+VB9hJ`jl)>7O5I>Anek7_E;Er)2 z>8x}9JYNW~!}(KV08UJy8VMDaiOI>ynHAD;()M?XG3(n?6@X!EwY9Xgk56f;~g1stn3CPp{-oxSzLD=9+z+*0&dlJ}d$kFb`?b0Zz|7z+jE z;Efc=!?0ibYI$@XHuV|=_FkzZE$tDm*4)ym**88QGka;$LgCETZ+i8m!w(8NGK^)S zUzdX1_vY$L)YCEb_vjv7i0pYm$Nt24gfvYDcm&uPm{6Agc|;)8{QfAoK*#w_7P@2P z^j|d8mQtjKt2jv$|MT zAf%EgjwJi9dW`xl=2tg!`BJXp5AoG5+Tu-rx(_-JXN+E=u74BD{*Y_I;zek^u{1SZ zNmOPi_;$K(Paj9`tp1sxnZ-mOx1`ad;Htc$$V{!Xka2FJN;f%8u9Qze%&w%=>?4Qq znP7*g+f#vby=pd36+Yy|+w>6_AQi&ySg#)Ia14~xu*CQwJDkm z_>{z)a0~;+@On#lMd|1i4D+QttoazU`gio*4j!UDD%&(+w7&32z@s!bWN82CqBVml z$49Z&-9?vgWZ-&+x}+gC`V574z^0jB^CqJJ2Qmc}d?ysA2{_x}1?*2vD z!oeY3=t~7219SBbSM2dE{#j;!PuIVmzzixcQs}O)@=2Qj<=!bwLA((%mk({Ru}Bi3 z%wibo%&4ojdhe_WY)zb$vhavqQ^7g)yeIm`XV&ef@V2YR=xsbm17bx7B(>`<+n`p3**B)xcA?)-=Sgkx+NOo(IZcd=UR)!pB7G0r}Rt?65 zvsR+~X6qFXqYL zc*rg)t+E_DPLB$azMQoMLJm!R1VI{ZRQfnVFvP)<{@PMkEbRG@M~xu*8gV6KL=pX| zsO(ch=#Zfd%2ZV2Ra|k~eBRp-u%=<`lD1a<^N|r>k>5X0NqI;jDXzftf2gg?V_EWp zEIGcB3Ly!GLL`~#m1r55Y5xB8qDJF#W^`YB<{Z`_A4s6Osa~C7$i_TN`$()x-8KC+r9*&27lrXG?VDUtgNhje8ha`7Vo1}$F0@# z0RAeWv7v{e32o$1#DFINt9JF3e-C;qJ(H2DJRt`mZsdU}o`@VG_81VcRx8@w&g4#f zD1SExZ6%6FY!*Y1l^Y>_d5U(@l;IeiH=DxQ4?AJp>po2$=#O{-+u2hKAB-*qGnDY5 zPV{t21~wGylML=lQH#}X{u(Hmo;#SMW7DTgR)z+Fwlm*9W+p|Bh2ULiWx;wvqHTT< ziaE)?dXWsU=`FOC@rC|bI{Zh`p2#Owk9bMH#IB5vEJMk_XJn3Er4#P5AzH1*$x^l7 zqiW5RpHZ_zP~Pv^uM4E0DIb4F8gQeP=;rL@A1l-bzdYmE$@=2x3rWg3fWf_+^&dicTN7TA<%0m1&I2Q4;~!y3)W6kW2tr8M2R#F5}j^^(r-M@{E2$5eMJjI&vRB zmgi^4P9d-Q-Vm!pZN0i(OCUc9+WbxEL*!V@VSOK6w zJ-9mtUbu`ZK86sjc10u_wQ27j15FF&dEi+WD$5yn)|t{&-WC5 zo{zq-umoz)AaHBv*9E$tG;mNJg$$g9%ZjrRAW)!Ob_WE9{DD}{Jjx*ziC~YIXs@>nSH~S%eND9 ziL|MlBQqPDk~6(=ncmSA?5KOD>U1L!D3(;+B8jazsOmyx{ISOU_;$YF%-avb7U!Tn z)++GaTr_yrW+?5bZ$tYLBf@8IE;TjP2F+Ip1C*svSuXwwXuCsuf&3qU=gawKd$Ro8 z(brc(($ze{mvj8>Gmxo{4vHUI1VSAEVus`pd#N|fXGZ~lOcev>aOZAkSvbXBdSUV~ z&iJqq9^82Pkaf7;8$3tJ(jPuoU%@H%2VpUj8(wZ7J{JiYZU^WKG+005)o=d;j8=?} zVBYsRDY+0aVq0cnSk&QzDj)nLZ~D|i7uiE<=f{#Ry77{hM%1v? zs}n@uR2zbQ8*PFoh9))Q8rNCvaf&Sr$nI-qP>gutG1>Tg%WIPlw7dSgR`&0o|5nlG< zL#Mu*n_wTn*?CjMN@=eQKSnw~S%YHSV8E0xz~)Xsw>+WjM=lZqTiN7%vAW`4iJc=dPsZArFoSc&OKZjY0 z3nqXlNyC$5sTppWcoK0YckCU4RvArob>&IcDzW*Xx-8m$m#c?=zFBKU^!QZyML5F= zk0dH2th2r;(^d5;kA^uewxs62IRZQhE{6lj?ux&sZ^(X6u$}0|+$3ewYF53Pu-Rtw zwU~`71_+W*Sw$ClpB{m3dKLi7;o%Q?%6RUrec!bf1JubLDYqay|8HQdg@PKm(G2m! z&e-NthHJoQshIu#xXWv^)X4WS4={aa@t&PP?Tn_FAD-r5bE?#Eo4q#k2N<`?fWQ#G z=ex6V8pPhf1ovHwD@PAN+cwK~l~ZW*LLFX-k8W&igf@>m%-1(Io(M3-=VU}{k6IvO z8?4pcJ6=E$z!;d6)11gwz<8lziWyMkO3kwPGm?{q0UXo;TvSgL zb)DKW=6{GGP(MTuxlT|3A;4~}Z<(?7bL6n6Oi?-lPGfI354Ymh0p*>`#*PCbi7>GE z$++`^xIb>&K`1S3x~?eP8cpO|HbDKy{r0DcLwF&I()~b6Q=gB(bAg6O^M`9vbLNZq zE$gbAdFw2=mR>l8jcF-9wYl4r{pM!GU`E)5lg7Ukz{Q*l^4pdPpQHJRaqk1G_Obwe z)Tfq5bm}wx-8Gb~ulj1s&yG{G_0MM%$2te4)rj-&a*^3gu@*Cg01EBe@gh`@}rZzgl$%w1aGJ?KnuJUxr&Euy^D%Xxa6x~Ux_}UM^ z-_syJL=W$jpF|@P2_yL^yNNjth@ODuW+?(2p*yRD&roRtm>1H&TE1&|+Q6!whVLQ+ zDAL3RE@@z#F%r5k{c(z)M@(oK;pFNXd?6LP#}7r)*- zDx{>D%bKTuKO0ztd{NQLATdzSQOg}-nkGK&{n`7-ex{m-5!RlS0D02*UeZUGIaSL# zb|Ysh%O|S=HdCWcj0mxSE=|vQbt~IFV(1FDGE9>f|@(}!degE%YOBAD>V%s>M`SnEl3eW0+O(jhp z>unwIZvZm?gd`o($s${>zTH&cx%*!zo4%hOL^~}5Z2=N6z!R9YdOHnMVf;U)zA~!H zuI+ZyASHryN=tWlmy~pulyrx*bO}hObV_$4-Q5yWN=i51!sk6>od0{=d*zjL&IM@% zjJRV&WND&CmP{Tecvt7Ox0>twdwUB2BX+dfV&d6rK~8ZRdBfSmFe713=ctt^TP?Qi zZQAg7SUy^4Y^#v}kQSSqw!B<(B*|||(3YE1-r_`FQn zOa5?py|Zvcy}jvfVCN4aW4u5Rq+tyL$pYV-VFS;%L>j^7bQc#F&(!^3M9NhhzEQz| zuU9KYDP9+=B{|k#lag=*rY%O0)zHey({>=J`w+!nH|5&$5x%SsmgU63I@`}H#{2r%!Vo-C{^{2ArfZ9cq)ocm6 z;B_Z}wpf5$`uv5M=X%O(I`vlICW$eomQGgc*W6#D$8RjQw>{j>cc3AmLd$lV@kyP4 zQsNyl`%lO%Fp_|`Ej{j~Jp{Q0e9flaOs{&-P^|Fmw*KBGSq?M2@Z8b>TkM{YD-|Pn zHo~?$o^_QF8$1Pip#Yu2-5PAZ_1xlp_@i++#TzCz`1LY%U+B}q#Qoo>z%=qbGZe@< zXx1gIw?wd@{2M3eRhhNQxwvqk2>j$+BxVTS&CI>&v2d5v*4ED6HMrlz3A>8zRowD9 zy|iVMs{Hc^inO>xgd$A28FGxejj>5d;A6=D&H4FTk;^$C*i_cnN7K7;;hE|^xd-Q` z^=ip}dvSL5c@6D+@0J0?kpI)I^EZKC(??1=i`$33%(_mnZIQf4^lpuA$3Fr0KJT!6 zufVEyd_ zj-WwDre?_A3l5y1OGRXkE$nTE^PUn1I`gq~trpjEUI|6TanS#1X=(Xix9++4{Y&L> z10jdx{(H{8#0G)-&_-%O=lp!J*n9?$3*ne=OlCVZWWMhqatlOi=~8Y3WkmGPDQM z9^6lMmwDBidmghTYM`acD#W`l8iIq%@h%B}H%g~_u~x5fNUqupRD)fI)|*}Se*)5y z^P|(GGESkiGMx6;??4@UAeWwkKtb^>aKl%EerLCRY$@LY2@!Kuej|y%6 zBRo?gIhNDCXBnJe%Yu38cjx4n+M2dC`S>W(H!5~^)qcXD^!xtijD?(7!1J77?K;@t z$X<645Z*x*+f2Y4@IUbJ@u%*Yd`{N8aP-}~3|bs$07i=!RQG*{{*kEV9Jdain9irR zCgh%+_Kw}6tb5L=Gx~Zxo@ytUun7uxbrRB(lIG`Mb%2I7b8UlI3?`4q=@w9LneFTA z8?X)^P6DTGa(X&`=jj;mIe@Ci%GK0gs_e5=4(lmb7m0t%gZF^FM`9ZE)04B-Z3t13 zgixf1tr3Nt&k0uV1|$TaJ!0Tn+jLlDpi zvo#{aeR(FnyU{*cMQ3SHI?q4fvPX>Ff1MMVm!UV*i#e*jJJ&-X-~Kgi+w+~=*!`#s zd3Fq4Z47Y)BeH(O22_-%3lvy1=%yV*_UFG}4HHGnXjPjcCAGJ=|I|B*HGyn*e@4#T z?s&{4c|Z)lblv+)in8-N9k{k&-yi>5u{^T**!_W_X9y-Y+idXexNUN=|E$)|v@)K6 z6mtQnk(X)LSry4KdQai~O?-L zoU%`2>m*>}I|BbhDPV*#F@3MEceUv2yd&Sc-|TpC*fM;p6Z3}0<ZyIJ3xTeGyfm)kez=3Mb4=+! z^1WH-ss2}wGW~sg7P!aaIhvE5KQlb~20Z4v=^IEzpAbhpZ%)k?sz5P|xuzetBApNS zcF_05;iEml-0!LI{b{v%9 z3VRzWSyS&vC=a-k^N=`m|+d?RZH-QsBeF=E$KT!0K6vs^$D&k;c z17DNk@Xo}<;wQgv{}m830ESBphoPkl(@mVBH`qM9w)6s5*w>6Nn)o*zzy!Nrocw>c zeIH{T!;OS=ZO5lY&VpXqX>dTU#x;z(h6a;f^LeK~3?ahU&Th!i(5ulGp2Up4F=JyB zBPoC1VBtu~{|aABiPM406IAl@*3qG1o4#UwHc63Acwfx8R6e-R5sIvT-DGUYuLv;m za=JJ((5~C+$S8Jq=eMt+1Hk|{xINd!$CV61-A|ibqaSWWc@H<>cF<3!Oaq5e9Ppqr z`7HR;+J3y2v;|o|=;-sg9kqgn-UxS7Q%*?<24v3%E>@wH;Ejea$*Q7YGFN4LU0p5bQ>qccT|-z!jyCgQ+YWOzF?%XBO<~$`e=(bnqpGyT`7TUPubsg2IeHn9`;)XP9kp?@A_AJtDoaBu4aDh8F}HBtNI)O825&w3*FoEzfZJMS$xDxS6h3v~oPd2LQftF75kAPZ z6i`x~)YD?2uTQM}+GqP9ihycxrK!t`QRjoERs$h?%~Wjdk5&hAl|)HIXp2A4(Gi)- zX`49dN^uSAv z))D0}Ee%Gv$!Ygp=FzVp+AkUgpDw;?=;NVt+^^3rH78wDKePJOvbXOn}QTNpUzND1A-dx52BH-m}m;MrjS5Z0*MF9bb!;&!U2YkLhcA5G4ebV%5hu-42TuV97 zq}Xd>J@2)))!&Op?p3&P+|X+p?k&D6J2>L36Qc`;W$A@xmscEf6(w7gwfQgL&yCwO z&=K&7_EcDKWaS* z-fk`u*kPC!;Rbqd8JLdYEbu;2MquM*Qy?sAK{=e@ya>V9_#TZL$AInTe2z3RU2xGy zPtQJv&H|NMjm>xv6{c=z`v#&Q?cgQh*PbCqnJw}|8f}zH9gt!bV z{f<@~xy;F2?xPI{j}D#S%~{RrxbwwxO0cqmKn~vOEA>3OJ&^-vIsxo!!QzuFK5y zXhlB;4(-QKx^@$5_o8qOoW*rx@gQMCorHI`h>EZ-^U&4SAXUK1=g*lx$O_~Kz(im? zIODzhA_x9fn_<8Z!s%ezR zR!R^RzI?%`Qfq-IWS^#Rl+GVjctrh+d04S2&4z6UPFAjGPIP{;OTA3Ap_4meHq>mo z;e6ZT0aPbELGD0AL}b#^Qm)tH%0A07Moj7KS0o@imd+yv_n<&&CTakng_yb3_4Oj{ zw%V=StVrmRu)O|4elf>$bBH)gpE=aOAHi8{3Rh@p?UcolXLCYQGi;z4BCNYmduM+9 z0JZM#NL)PL=-n!bpswI9f|5qvgrozCxtqqh#EtYV)l_f+>sY)t+NN-?7)Cc3O{B`V z2mECzsH=5~UT0_7TQ2;x#eWHR9Xpqn_3VYgq&S9cDF#n~_jq9JsTLaK#O~Sxkfp|+ zWCGgIw}j)0`c4^wpFDfRUxQTMq_Llf$I%oNaV$%84P##Qc`fDUk#Thi$QCUvw0r+1 zC-J!^D<4fh+ndY{9D_Z8iC)DKIx#Tp(@~5;YQo>Pg%^q&+7qPJuWP}L9$IpwTemtA}DG1Gn=J{Vt<}X*Bw2JH6b7q$s)#mZU{>v*BCFbzU&d&OU zr3D15Z|ia%loAZ;tkfJeh^o92*vsWdi7DSxGcnn{j9+M_py`hx86O|_eZ1KXu{KP} zB3a~l)(3sY@yf`@H6Qd<5@e1Op@0apdI!;G%Ch=FrrxM5cUZK;R(jHh4ru^zhtMsE zfdb`0x&5;U3;>>?=(sNBZaEh>1+ZMv$UJ@ zxRfj2IxV1+4){50rRr@@k9Q^o1VbAeAUk{bF-=T~lGW{%fQyh8F4J(REb99%P(W5? zBY|}+2;i~5L5oHvpLya(V5@kafnXTpj zGK;)Y3cJOqHTE=0#}Hfh$HpG_;-d_!cA+EOvG9mAgH(QPz_ToTm!S>R3N)AcVw9nR zum}TLYWJ;U;;u?i7^zLJhRB8ov(JW^I?^L46>L|SBk-1c)5`fT60|}U z)&eo7mcAyNWJYCDN#DaEzghclWhn)+7OkI5mjycMRZEiv*$qh4H$|WT|6vQbItSt$HMIE#p8+pEkzcE z$s#IVCRwJ4;?(Uy@WxUqPv87a#_0xBK~?W)!)#fhd4Dt!2*QAf7~!X14Zzkh9^GH* zeLWk=W(0R{N`1=HI(d@(xc!ci$O~lKGOI<^ZL8dNu=TQEI@_b7zSv36rxxKb9$X~u zKw8@!$log&?vTdKZVi6pAf zFKqAl0gwJ&vuYER&wrO111ozQ211%a>~c5vh1Y1E8m!k~2bn!a^IiU{ASSFnHi$`~VY!Fy@-f8vx{4Zp%bp_=E;&&mcj5uap z(18PQ0|rzwS3>1&)zpIP)ff>O4REUSEuQB3VVv=igS1%&>07*VCi&0S-wU&$?WM2I zqhFvQ2IK!?q4NQmW#H$uG1CG21$;9EG`aZr4$lG zAksQMUmv#*3mf~EdUHLmrRA;{$H0``eQV$g*12h{^+tD)v1qz+S0I8I4-dTe@-nK0 zXl#xSfid!TxQWeo$P*_OIxiiz^xE939-c01jYN^u?aB5l zunvV2MrwfeJX(5c1h!t86uQ)bQKu^m--%HeS*J#o_jvqV$x_i;GVk299T>X|jZS=(_e?buU zH6bC-^E00t=x&*JhXjfoDytY67;su-O|!>>bIFNjHXHo+Es*^&zeeEgxcyUFym;T5 z-wV;&WO}Hnjx=)FmW9cVz)rEN)V!acHKQ-U!-GG-rVoXhU|G#r?SxDMfeCCpo`S{{ zHZP&s**;ffPsb6h+Vb+KAI^8V>l|pp``JtxCW!#|!S!{vDenhvH(pRSE&_J8jf1Ty z@&qPzA7e%B3<}EDwX)jwlK{ZZgt+9-Xfkfo8r{f1b7!u)5peCV&uaB?4o+j0@i+`K zPsdtIf8l1f-Y=i^y}&7A77ftkKdFg<=q34MHT3I3x^+dgk1KfrD@TQ^s;TY;dVNsy07{$no3b&Zw^lMak1c_1Yx^?tf!}Ec9AkK%rTJL zrV--YaW1HyX5AVX9$v08ks8vy+1SBjdlQmAXL6KRgmraBLCPj`aYN$Q@?g>n=hldA;;%GgbH(H0<1-d@jY#9;f)~!M;Xu*> zyRgh8+|x+$vH8N0g_>H~FF09Gkg61j-DU-+re=Nq3}4;08;r20>`=PTQVZK`X^mo- zZ{SB&tv}Qgz;De1Pr_4a_(J^+e;iUm#R0_-umaQTMkqS z>-ZJWUuzbq8wcc2b4k4a9Qavl6fRgntie0rE-q_yU#}dx_Cfgr5PQ}$NJZo*Gw=TL zxsSE*TX75i%QqTm&=#-BT8Q!-O^xg(M>lXjvd+>ma41Zq^G6QbqAZ<^xKJZm*>4nG zWZV47OuI`2VUeGMR17guaO(jvW#oHR5X69p0-cbX8DA%Ja+vOW>!#N_rXqc0#L{DZ{zJH?xK|LQ&!3?9-_yA1L7L}seZ%ot)7MB9uaKx=S zCgG&SRQM!gTL+-z^Ai1UM2w|MIJt*R7!p=vZXP5xOMLJ5U*BgrIL2&-@v>Gj_PZ zA0D5e*wOXqIvaL~hTh-nt9`E;@?@4fe@_);Z+%q_Y4c5W(x79#$HL4$q#fkj4$jFh z;qZEDCKIiI)G7%^cV1< zt?TG6LJ>>R5xoQ#DD2_anA*dh7Z)?cw;kB(+Eabdw~jK7Qq|a)N=}25(B;6UXLdY; z$b~+tXvi-O`e>Jj%fR50+|@N+(ipv5o+qYEC*sK!w0Q3IP0p@^vkbZ(rN{0ML!i$9 z3gFMJQXmR5gC3vFzIWJX&lHO|J6KH+TT$^-xVMSd`{}KMK1#A)-GC&i5(`XSy%Wh@ zxrXp=^7bVi1>Op)Q<%my;Nq}jXMQhluw#CKF+|ptKUH6?@0S=D1p9)5huf?RBYlef zN9VaX@kY|}EfDsDdgUlTKM7ILVjKhJmD}brhG9~3HWgG(l18e%%i!H^(QVB+ud@ct zB+X`L+O^^oTHM&pO?_YZfBM4ja_%2025N;mL(!$J+Ppko^+Z%sZSZFT#OMvp*~y%) zaCXZ3ROx;bpr~eVh;yzaZ}FYUZ^zG`;fGP-TGwA1NTIeLc5wLD!6yW~FN3};Q20D* zAItJA|!IIGGt>&vl*;`aMpu2D=J5Jm%a`ahsUn43TN%u55w*L4NXZU84cyS)N93Lb#B zgfW>@J>^dC>~d4O0W_8OFff{Kpc7)>Ak3w_p4g)HpU~IQGBv>r-b zS~M7n{b;(Vh8EyR{AAOBT?;J?R~(CHio}=;g8+pU+<>LpDI0de=;ZuMcugQw zvNdfOEGs!ZKlrC(O`MIHltcfV9DMow@av$C@s}3zU|aiMJx8|y7rNexfNbp^>nI!- zx>d}pEs$e_K0{AUx8A84D9cDi6Uz7spcC<)Z$>cP{`@lp>H?s;n={5)ja-uOHD!C3 zU*I{@_FNhM>eS{#q`x_kowQmRyn9_So1LXXj)SpeAkMyGIlzf|MxRi08Ep)-KOFmJ zd!B9uqY~URNSTC@OGW#Qfv%x3?*XNjlHndtaSi z!#TF-rumk%5*&J}pjT_Sf<9Zp@YDDXK<9TLbtvcLkISA+a1!IisdM_)aN_69vQ$hb z4g10F&lnkeyd&C^T(;L8U!I{9l1lIm2a<1s%{TpaUY^9Tj&9>^&Y1x=6K5L!C8a-+CeT`~ z9d&gj&JF!^cP}*fk9bHKsNB=g6$uC2)j4k?JFJ;D}TSe@YmU!BXu#@(FXSxPk;i~( zwc17ZVc}rK@U2b!Q4`Z0y{^}3mqXUzcGK$fdEhTVz^ktGcfCY`Dk+SfD|!0{D2<5h{<`_Z5IvWM4v-=d zX(82?a;Xv6dhK;mVNG+d zv;-{Drj7@#Ee#(ICkF?r=tgEHF_xUK(5We6PkEAoPs*S9nVGy8DB?Vry-G+>*5aXcXaPZuWS8k5l z8<>JQ2ITb7(WnG7s8m^6osu4eh1Z1ST3UTRD?f=hK2(3xk7c*q9i4XPgKqpmoo=Bn zPzR%?U)f0{mzGRZxi=@-Ly>g4*C%3_-m`r+Tnsdfr>9nJl~|XG;CUOdr4WBfr8IcB zy1vI>s#MM8hOp$~U@J8$SX_~!gu1?Q?qbC7!-l}!E69kdo9;o^>i=jNE1sm=v&^!= zNq&cw8uK-kr|YmQo*G8`d2w&%QNJ+o_;quuF81+rP4nC>KYgbiPu~eu}A85=!duW% zLr~PW!MEcFrO-+MbgZhZT=!VdAq;;}9HjOpFi;N3B-G`9) zBnxT}Y5|{~jN?^+e)?rT=zb-pZtM|`*fb* z$nvokS3Ev0HTCZydE4(9s_4Qs{dV~JJvMXLZu?;^7_kxjUH(o0G+Qh{1eqvX?GmhD z9n@>!wl@0AG9xU!<)cG{#&(kD_QzI%fyRqhpW}<`G}2zus+ij3B;7d)(5J7jr1OJ6 z)ee6%Qpoi?3{qPciV2OX`f_E^zwvBEBwiQv09TI)+Hqi8uq|vp7}4UTqI~|uU(`-F zNj4b!BWX3e33v7-kD>cx?eYvXY@z3RcX;Pck`D7VRgP0edV4 z>N1_Vs05r8n<jRrJg_GevBuXu_JzrM0cl=15{y3^t_@r4mKEjXPxAua|lp z!eWN1>4;=}W5^WEmeWC#OXK5_X$}q_NGyzvOJZzT<$h`Iz1BlLpXlj9`A*Z%=>jtv zEP&TO;mgEy#)%=M@Av>Mo%Hd~g0CB)YHqX{nyrEvUs zm}j@&tJYpxJKXGy_v>~(0jk{!~E&3G*$~tyGCp6hU0}3EVcq| zRAJZ$;3?zmI(gNhYnR@jRXh}^vuoz2(7vXxRk8=g3Lhe!Dy!T0wnEz}abSf)LcMrII~5sNS%5L4HRU6|=+(}bpj zY^w*GA+MXfBp@x&zZk~+)6w#~_>(!jg)qNt-fq=FX95KE{Tm%tZPhfr!Bsr=#cX^x zwKi5{@jR}(o+S{b=P z{1GQW`8E1gA;uma3Tj+t4q7(0JyLLVL~U5=`e`t;_B_Z5!@w?v&aiSW&bc*n6a?zXH&$z*5d z4^ujj4Bvipr4e#hnsiJwg10DCyF5jTWko|~ylFPp_Nzdvi4+kTBPM0Dk6dDFWPI0X zq=`O}omIn#Sq8xjj>XUk}bD|*P% zHyCb4G+8t<>{1j2zAVwfVmz{A z9t^h8uXbNDvj_?bGBDV{tI*TY!HfaLLug)bR9&djQ(CDMq1iof;{!vovep3DZU)l! z?sW|*FnkNm3i;3A*OgX}8YC*9d3iY_KMOrk2(#y+*e{NVf|BP-1>>AQ#EiHiCB&kr z`QcS^OLjIX7uS54kmtH10Qb7PbC1mg|3$T-j3v=BO?o%L7I6{qjy!zR+>;gYtfppM zMoSJayDL?gWu4Rz!x>H!wp|`!qV^A(yjT^hANO_AeQ;MnobzAXinK-PkCei6R2`-h zJWE}ZmKNqb8np6!EQYOzXxPf(yowOJk91M$&&SdN+x>1gdic-_ifC=?)UC4yW1`0#_ z8yl176t{Fu_JFW?jiW1Az`AJky{L!=dW}KRTo=-!12b_nl{C%oes{p}z%9TgOn?Ia z2Y%0>bar#K<0h@6$?Z6^#%B{~dSGB+{QmvhwN6E;q)`($Q&lxdeF` z<#JVdg-~hY=V}{;hS&`45TMOy!&R6B6IjA9x{x@0EL@dMc*62|(n3D9v-&cMHCAYm$)L0Jh2Do@C%ynlZYQAhPfHFW9JNJ#ItC}d$%(%w+&_e zI9_~*3;*e3;`jwNYJ*)lT5m#Ng3}X?RSbQS8y6n|x@D?Sjil()N zjsV+1@X`stG4ppKagA9^vxD?BS}7z=-(W(@UnckGDR%xv-90_jEuEkUi#b961gG2F z*ocCHG6I?S`ST}KiQ4UtTvcfH-Vf>_$DLK`<=V3g3!1vRkqxHok=50#`=7FhGlNjA zZEYp(X5TG?U-D>b@P}1G;#{4a?f{Tnr|>@8LBRr!x|h0&3lF9Hcy9Tw_<(>OI|3^8 zg8y57P@WE609F=lwy9Wms~ZY`$t4oh;& z6hkKziI4`q%{ z%pdA_ATgQ10MTagaL8s|62Aj*&Hl^hM`RNcu$1T}dZcA#yY&}@$<@L2lQWSz_$)b4 zi(;S0>O8T_?`fwV8j*d1(z9+ zw&wWS?o+;jJh6oNk=nboT-Z-u6MY6wEmIQ{FV|laS}HM+mH^|60K)!QcTN|I68ZLQxw5p?{ zL$~ZpB|JYrZ-R3WsyFjUwg>M{6dK8xb7pl*0-hGrU{Mt2vOfn*8%+}G<;Txw0!};f z?(Xg$9!;MzaRl>|aWT}tl|>$l{+py!6k4)aoLYzifz6UA7PBeTxkC9OMn3K0W!K~f z#1=Bt@pzrNucwa!j%&}v zx7MpC#9vpRpN3l{Uyd|uuJAAL*hIf#K392;KdfwR(a_Q7nWo~hGBY#5iqPhk)YaC$ ze@EWR;pPW#2F-Q&*7xGTlpWte;e-r4Nwev>WKJzIf`I;_hpn){qzX4%tE|bgxsrg( zH-D>4$+3gUo9sxJ*@^AOk{-gPn3S6**~257u9jVKeeGVZ-a=CqtE!?>;Jg4NkbiPU zonUWcqz^l=m_r;m403;PfupP@kQOYrl-@yS#zZqQ{0|w+^!$sw7yg zll@ZYc!~bC8j}CJ5mD{SKGkFW>oJWJlL$h}#o=D7j$u9?sX>+Ly}dn1H#_Y{|MD><{b26|-orC7$z64=&X@SXe|e#n39DsPrHL{@KUc=Va{yoe zbl5>_?qkq&B4%`yfo@wrpnQ5ny^ z2!jLY4lo&XZ{Hrc!fl;YN)H_#F`f#=I5ZMHAB*19Qu=ON?|co}M>`{zu^W@^@~55; zD`jKK0i2YKPSLOCBAdOgY{4^NV7Eo%7C{4OdU_h?lpU-ZvyOdhNdLqHXv>L;iaPUV ztX5zJy?rbGrn${4Ylvgi_}^|U$_B4OTI75w3J2=0n570^^B{ziMr?+X0s&=R{-XZ8( z+t}zO_iS%%(QAq~ef#zeNWAh)n1hR#J!uh5B4j&G2j}J0%PQjL_N6d&I=Aw9@e>(l zRGExBzaG9a;(7!-VE0{_@xfR2XI7{o&Nw@YYIK|q-t-R5M)q*o%_K;OfzmjWL3=Bt z9~wVk9;eeTsJdanrjZMI6<`2aSXdYtE1H@XZTXON;n_jYft;t4Eut4PYCNh(4iD9S zZ2L-bGyKyKxc7wFT`eRZF)0M{GjLz?4u2R@W*vC{Co3Bp4r%x5WUdqe zKK{-f4UiD9w6whR-3EL(us=OLJ*6cj2Rs_aMmhEih=@u{+>103y~chYM%DCYyG2nh z%NMN2k~MM6(OQ&qQvhnQz#JSW7r0#-BZg*4DXE(3YGV@0z5pe8ta&mYx#xjZO=;&UPko@&jUvk0zZz^RQ`cVD5ZN10@Y`fs;i(x{_ ztc*=UN_sDKGc(FqTYZKflDRDW{NDO7$_;gO5>isjhZ23S8@+E>E1cfFi_Z9tg@X7p zc(+@$olW!KJ!8CW7R60-S`FH`Nz-m8rr(_g{1LE!0KvxPOALp@s5>w^n$e*OdYJ~e zNk>{m1qG71lX-c0^xlkVyP~_z&*wxMLmv`>BZwEZ3 z$hlzCpEpA*Fyjq}OirQZ+0`QL*r!qLRI864QBYAc(5Gx{ZKI>2GSH6@&fT1y7Y5-D zj*ppWXgWFQ85ru8zXn8({WGf^Tv0J$-abig;WT*jWe z;8-{n*Htt$k{~FxFhEFmRCsq~Mc0u1e`Dauy3 zS#J~6a(e1mu5P_tK}AIskc0COt_Rc)-@bie&b7t@g_rT46=Z=`di3m{mNi`v9bLGdGa9}6g+0xN{WMKpN0-0uJ`vuNk#R4!`g2G!{S%;SW|Sjp6NvzM+Nh$1A85lW{TONPp}jj5*+-| zdfX>^u(1o`ijIof1|GyFs=y*7z)Vr9|ESDqqjGx6T>&-<&~%+35I=~FxX633X#z>N zslCme-fmTrl@H%|200^oJ%7p+O}YbQX=rFDAn%N{ap-t309I{bVIk8=K7?`KnRf&V zyhnY&e9Ha-Q@ecgck##Ng1w9TQL|?QU$%?82`^6AeB~`J-y95!krmx*JgNn_K;Vjj zi|zEWU)_OihrFPFq$U)Y`E0hLRp4Ma4BiFnO8` zvyYA+sy@>BnSA{C5flb>@3B9=YPtcvMF$55WYY`n>DmXZ=-5tW^m_HB~p z@bI4U--zhxIE-H|)~~Hnu-b7^M$oU&V_W;M^Yf>tq#&gp{QjNdLhdw9l$@Bzspx^q z;0&Votu50i$?1s+LvxZK72UGy+ZQ2GTc-Nt|6RMe(o*eWlvh-{;cY{0vfu{Kbr-d^ z^0Tv_g8|hnuNpE31qF>Ph2S1U=Z;_f5pZS&Pf76e6H>P{emSL|SuFn18@7PI+(5^h zp~(t9`D~c!*jGwdU2u zj;Hi*R_f~Ofl(aa%+Aft1)FkK8;Kt854f=>CnsfPW$o>PElp!@>Vq2^Lgn55V1?wH z{5L|58OrpuHLb$EdPKKf4A(1@ssmBdF<`igii$u)A$Z&lHlxNKy29VA#NLo(L>!(R?fB!Ob3pdgalxHha=P_I}-RT#X!GMiep+k+0cv&5f|KGgx zyWU}RH%+M0{(UWyA%$=qSE+ef26UpKgvZl)Odgg59sArF6~n-(76-B_mCM|`vzJA?Fc7}p_gZYB#rRo2+1dOj3GYsZzRpM1w`-kW8qAbvIJeLEdk2P1HR(fUKqLa(T)DGtyW27GTOG^VqvPW&xf4Fx+(}J+{daV5s1f)= zDaJf@YmCs=L7$Ogo<9ffuNGSZDxCoqou@A_+fm9WgehUYj_zQGRf71e-%?Xk1BKVs z)YSa^d_Xl%Fv&d(%&uL>J>NeKxM0+`R~0S>8H(&px7Y8uWcS6PDwY~FIU zL>ZdA^vlfWJPQN^)uNE?@w0@hhoA;Yz4Vh6u=(#iz7s@&#lw&Kuv28O9_Jt?7RZq< zkl+0OxqEE0{Pj*h6-}DRQCnMC!)(oGpZ9!I_Gjkl=iqi)A~*sLae6&Pup_n)N{Wlg z95ZsFIok0+4BSw&&9YlFSXbA}kBW{0=2l0v38TH>^9Q z@=Fqut{9*5+^nbjU2t+h=2{;jT}7YRW9ja$SvuR-->-;s;jwe=43vIB98mda&%wcg zA4Nq;3Afc2Td1W|YQY{}Sh)F%+&g2jH>KB@MVKizgNla-uQ4$x$qhW^4H^`LiciOy z5pTU=85kI@xleNL0h2ij#O5F(F=3D3K!*APPFkUE&-~mR{8k29d--_7)Y6jY`JQs@ zEpTTz2d&!L3FK1ZSmv#9arTV&$~Dik`>Xi4O)%mU%0`7JXP{~2kPEDsc8q2>y8z*eYy`t%8awG?Dzss^4k)YLwA4_*M( zICC`q`L(sIv@|*RP&iFmO&vZXJ9sV)jWemC=Oc_G_RMJp&xTLenEVF7xz0wxk%&K9 zWxGj}7O}l)>AiA3xIf6p{Za{oeip3CYOZ0^54-&)(jI z1v}XHTIMV8-aVWO39oag@p|j!lJ%D_U)tK*u;;d)>ju5OL;{|*b#*R+o)Ul4U|`_5 z?rvy{i$_ME1E{63gp7;_pzOyQ$y@ZeZSa`??OVVx9kw-EEpK{aB0N4L6H^vzf~mdz z2{?x2r+Yx#LSI()MRH3;#n_I(??10t(rAyvI)wUjzy{Fs!S7|=Z&3qwh4CvFrH3Q8 zjx@(bOb{V3f~*4;9v)y^O0P=`c7foEl(_iwP=O4>cfj<*#K*TfFAW)jYLR$;e+DiO$>Rn|B@q@@LTaj2>DPdsp78%;>MNtF>Y}v| zozl{clypd^beDAZ0VJhCLb{QbJb-|R3P>Z}A#iAt6%WEIN?rCi(=DI_7(64kTK>p;)Sir#=jP%u-cVoullX&^sPS>Ik( z1}E@50R0SQVpyFlzsAZ17`$w5ZmzDb{u7ScOWu+_0C2?r;OpDlt2Mfq_S!FQAV%g_ zd9Fyw>lM!Uk%x!F#eb#BC^|fo(^Ld3xiY=#H%qMnf*c%2BZ4cNRd9CZj851y=D389 z&up{fL+d+t!EFPsS7fY)gOefEDCv}tE&ymPR2Rzt<~}2#+7*y@_jh+7NQ64C00qv|pa4#7%Fk^-ldl_#VUV{2#-8|p441T+(Un34Evt$MBD0s+vu9N^*XCl2m!qgcq-aVYrQ8~ecI zLh^o30~9L%C`8>+xHRJa{30Uc1>dQbv8AP@9T79|uqD884gMI)Ly-+w%^?;fBqW+I zgjIF6_8{5ua7#-|@b4@LeqkRojj|+L&5|lR-;LM2I36en^q`?!ocv8Sa9xrqj{u|H z{%SmuKLRYzV$usQE(+@fEQj9y-5R&1ZS-{JCCq{=4J0VB z=as2~YZnUL1)QEHZgXMoe@ip}fz^EVdthv?m~c2IOhby#*K_vU`}A9XyYTTM+g$=nWWeM}$hXh_n8uyoe%Igv3u?3=jJ|5?g7nnh#nYZ09c2SkrDW% z)#}>S8GdSPB!#L}pvL%vlo~|M3@Olz?X9gzFN4MzF!~DHu={zaq>81nW*znQXKiBX zTK;(&=^7Y7D_ma%J&!QL=GT$uN#Q9zJ)U4D&Rv(AN15g7j@W*GVl5jHLuX?Od|)zuZO?>BGw zl5YM3B(_MkMPG*`0XtDDQqoTk@Y|akKuiP8F96u6DJj2C7&J3pTS2^eVsf*<0U6qB0d2a)mVgt1Py+^}h=+#G*w+W2k7o=T#kRhmmCV&O*03mZX1QjNK_XE!d?>~P=lR1(pcu~*r&^#-!l;q*F;OCz61?1>2~B^sI}uY1ZT$UbA`R#uiJgk_Gz2D`79 zb7G&plSGw$LE$aT8+_z!6vwc1EksOay*|2*x>|#O%ao9iW@y=E&LVfoVclSDEveq+ z1uy?t1L$y%ms^LCB>&xH0tXH4_eVLdZDdqbPxhPLHzH*=Z4EF-DL^^X?0qy3q9vU= zEOtRc3#m3? zK$kFUUw&Nzjz1{|8%5NCB1*19AzLVxC-zB7B|08ug0Y_y440Kv=7&#Gvh`ph6=ILO zJoX6TP5n-%z86wMT-3h0ShGL$W8jM7FE(q&pbK7NS4u7uHZv5bw8`O9|GOFG=REl< zs$TLEi;P##R|HK;{0u-kVt-7S;2Vjdq{k{D9iDSXFhHr>L?BNCw>5SxqTp|sbps&o zl$Khh@bz10OPvDB3y1);x3dGq@~J0yry%Xg)YKHXHLr|$gI@nr@Z+EIOx&xBXt=9m z16mq{b8B6Url4Bi3_ndN*xcVCZK(R~`6k|9L8SLv$6PI^S<#dc;Wbe>;{Ip7T6K^7 zzv6Aho`^u*P8kk=_+MT}K9N69sMogeITaj1w(Bsr5Wl9DP1NaN-Czkh|M^qe0mXcs z_rep0pX8l8jV^LhMvIVHt(owFRiJ%0bZ+iAq*aACot>k<(Gn7bD#d+^lew>iFv=&?jKmRqHZX&p~q!znuUXHE3pTyF@ zn|2j9Bv?1(#v= zJOhJdvKmk_`ccJVGWjUy0HP>jAyT-Q-ufRQ``+DKE_^oJ+TCLx{-2Zh3-j|oUEq+d zHoL6@OaL^L@FQ;{unyLzKc6pSSmdPxD4zeti z`?I0Jbp<`t%l`(?fGjw)_gaM~U;kTRJk?;jpF}R>z|Y*QYrMdAyAh!+OJR2_bmlI1 zA?Emz74OT2w2<`vxS1r9FMhA^v909S-u+D>tyEOA!O;|68 zwjJHPYxMpqSBgP50>F(G;{7UQbZ}5fO>GNda4lTp2xvmEPEGfZkH1w^q&DFKWQ>oG zkF~LXcb8@5PC&~z2N6Ap;6~%MuOBNlwX)%drk?I%K$t+c=}hyrB2ClV5EEMb+AYC==i3pF9$Mo}4T~{46x9ZSKX~d>hfU zzMf;*s+=lne;#YR8hOv%mp@kr|9iC80foE)cd(rfBUf9b1bORqBnslw>#8Tbx>*pEo z(=lKi*xf|diR%uZ?m0j4qU?4Y^G1Ra9QQS8jGyuvUYQ0~KQ?A|Xz|I)BRZ8-=G;#) zBoWEa*umgHy6e4K_wKWE&{V(%SgB?5H*9TB~ zP;h?ziWu#4Ncwi`spV@_`(cA&UloF@I}F+U0!ef$;3}xKDrzQo2?-m1=y=H9ON}+Hw|u&LpY4Ki0k+^j4!NnDHQP`eORNBL zonLR1s|pM0RL58v6+S7kva*6YU!82xdPlw2N@Q~R+te&u?G&LDcAu9I$uSvf*r}_8 zmvS0Xd235rJa+Zh*|_s+9!74s3F4tyn~8&qp3MYd2Tzl8pnYsU-CJ$}7a}vf4js4O zZEqR4T4=ijtiuj}{C3_d>F|U2UyRJ^Z$29T&|`l<;wetmi43+AUpQTy#BibJG68ph zqw8B^p^$obm_W_j8*!R7$*&9acV&`eI@lc8^k4*c(S^Xj^T!)%c^<}^T)g-U;7njV z0q{6cU__{>s0hN}^(v+hF2TuUHlVukyDTdUgL%8_W$&cU_3MzV)>agrhFbL08a=jU z6q#a89zVI+nb=kVXWo-W+c;mJ1%Wet^7`(sG|6kEH>6-Bkbn&L^1=l*%Ftb~)J~=u zd7T8-ASDWYyJtB~6jEKW%BmOM9a|-OWqe9XP(9C>>AkrLf=E)@36lJRcljRyTym_uE%BAv=D-xT8=6=?U5H_wNxS3R}e1S`b?L%n#!{z6(wo zvmj`wxF%nDTZWreceT{3mr)qh?E?cLq4l|RwtHP&TwiLw}!nYVs#N@9>gL4}kVNJ;eVq=ACW^2r*!LolXa()S!7UMHApxP1ro7E#^ zZ#{$Dr_l|jIkW&W4}hvUIdQ}i+QB(RwAigK;=mm}`<}tduRxEDk^>ft**RnZT|K`# zdgn>^nh>w}p6LyN^oSmplpAYVX262^w7&i5)M z6^$gW*z`iq?Wh0uw9a&x$I>7A00!5JD&OhTO?=nxYdsj7R{vjT|>L4hng-{s+(K0Y*8_3EwZBV!1!O-M16{6+-Q}1ji zRj`&=UpT+;1fU89VgSE{Gf1}c82yfVa+i}rk7!$^()Vs}Ex~s~lwzAJW>q`z98wyv zA5o^@7vlC?=!Ao}srKpf`=g(n|GOfPYA0iP6Lj2?qNCCOd*&U6Mg_wUZhgt;r+7}c zw63nz^+~dR8qq1X*iQy#XAi%wZ83IaoBZJ84JSdv7hnSAZYypvymh1O){4J)1#o4TozE@tbLDdw^kljtx25d2P{2BB(A4%)wQ1^79-*x%bLuc!bl6A^GCxVcq<`MYCXnnR#>r#LFG zD^O0YlDEE^fsm<`Vf7N#2n!psUm`i{*oH~>EYcplaBObI%*1td)nXQ1IGnfa7WG%2 z^w3RLq$j0$#0ykadeHky@S!>^jE0d3)1=g3oIPN2btLz!s}9F9zEKAQ zJj1%i8~%SsFW4&x{;Z#hi^JXp^z`>nTJ->hnWh`=0bfF;v3)%a8&i@^?AO9c3?D2j zkGo!Pxrn*MIKsBkfIqRVRch`6w3?Tcg?`(j^If$q-bX^cDGPN@vc+V;(&}{R>6NKB z_GPg97N|PVjs&ji-@XD=9+pWVls~)sd_uHe$8`R~j3cF(1xc3MD0kYJ(X0B|E(l&PFW`flrntLdfs z^G&U=02BR;?yG3IaL5*ncI_5{)4X1Fl@~^~8}DJkZ;{>QX%MVXL%Hm?A!!V%Ws2tc z>%oleoMgjIyBsW*wCe7%5|f3cixSb zZX#yklxQ=Zb1V8qj>I1q#Z#sY046Z-uqDUE!AVR^90mC(8kOUF$Lgc}>f4*^R+a)Y zP+3|WbMrNywRQS;;z-${qsT+)=|$)4i;~xRUJ?mut-N!&V;a=VH@X!nr;NR4Za6jk z>N^+TF>cs#0i?(V_(RL=AOGB^ad~;a#oU!_UP`tI(xoCQz)|m9lOTZwTTk?Sff)|} z+qCPJKM|(p=iMIyZMC&s3=HlnDp5~XX#TW6%zijpU<+?U{aT=Sm1{j4Kd(o#71$3ED4fCIfd=oY)g7HXf9{gQ20yo)^1k z8$Lm2ap&RT>*veRv?D+I-}1$Cne(kGS6#hE05lY1|x#wR~OAjZf107 zLU0)pbhk!l$dH7ubqyH*{tQ?63uV!1mTh&-2}dB^659RBO!o2AB|(0wXYDgib{VnI zOkGFWw*|C?p9mVqy0Omu+UsmH?kR32{O=B8&YOj_r1NpY&vVLw$RqI^=UsEx;`MEi z@~zx3brI!GL`|ycjW)A{!jL5^v)rd73C*{;5kCo%fs^Gb=*NQIe$ROWi?ejODXZu)Nq)recbs9ZZUtW2vB@cNvxpWxus zmm-lyj&Iw%%QUi^_^ew2Bu7l`KkP=xTMl-1`n$6TKQ+w$1Y@*qMRH5aE;~?W=caV_ zfBR-9yZl3+G$l19ykAZ(!l2JS%ROJCQ=Db8QNFlP^4MOf{pk@idcQco#A9r^KW9nu@h(sUY{3Ufz|#AT#5 zu;J>5UDcM|k<%6W6@VV5%*|Bj29llj&vSDr(*TMjQKJ@b4-B>I=%i$J(|e7DS2*c`5ek-`&UqCgGaa)(j|r z&|hGnM+a*G#nl)@Jnt*X?|tCmPRGQAx{zfDsCQdCJFbh3_7k*7bW&${U9`+#ILL%`1vl!U5(b_ULoKo$7(aa?qtj`mf|d7 znT!M-!TmTIWu8b9uD&<**R|*4TSGp<`yS)3d)7S)Q*HM88&Hmd7cQpCLR?@kP@&AI zyI|5BF9q6_@6UL72^hA(XqtVfeA}iy@h|Ewuu-r=UTP0+?=@aqTayYzg^Kb}gn6>A zQ!vh4{UMR)r?uACN9doMs~vnTCgH33YSM9p?P~vEho?Q}#s0xT_n*7PrZ-^xfQx|< z6B|3>Sm)pR0On?6=?;rA2+d_>Xnf8jB$WmD8=1Yrg>kXd%JN< zo$E>)V2zIT6IyIexb&Nf4nUl(wTd$KxkF8MT97+JOST6)kvPJ-rxI&yaDedWb8^vU z+#~|OqWgCg@)pGz!&;=pk2;Mlzsv1PexQHIn2suT>~Qjr`!XI@>h-L{%2S)uM%H$p zKCrP`=Q}6$*6>VhU|yM79pN+@F?m!*+$5GzSBA zbqlbe{S>$hr!)KjX$EvA35opWX(&IKgzm2qMu>VJR-LYQTmeH2cxmsc5}N;BNv39W zdlu@zj2n?BMzFi~xd8}+J`>SmEZu>o`NFoM;LE{2F`J{EF?RhAQeBu^L$fju@HdY zT>>sFi?*w;7soGbI=;rXgP9A01+WC+`-rJ)CWFMt3)aq)Kvabe3{Ra6R>ZC!nNh=s~{f{3?f6C$6t2T~?6U!(9M`c-pb zF#3`xXtjTGB|g&3#rr<198eX3g=Toq7{9*sIZTINy`5lj2=6jTm=56_p7GB*6j{U zHS>#Vv2{>T8+ar4{#fLCxCPffB}!0Cs^4d4bVvfNU^W{Y48~jWqpLuhw%Z3>_`g<{ zFF2ASsXc5mcd`61n6bj+Ad7ckrB)OaHq<=n46}>EQe4@ z5XFhK9r3i)6t>r;0a?szw4`BCl;%)|oVt4VQ`(2yW~-ub%BT5Qu>OAJH!97H^vG$& zZQ;T&Ndy`{0+mn&_r}T#M6!}Sq9^Yp96D*jKgcA=&&8Asa~Ww?7BFxfu_9!*lt`B_ zwh=}ULlQQug6LbL#sW0vK0lp&_tJx@)0}LDd`FIx%@e^^SdpB0!qCoJne5dQqn(zY zBkCA%&nBC4|u+Z>`HL;r0v*PS9MjU**>+kM;K(al{d(`LT zRD<&$uhu&RP>l>jF%d4HhSbqe>Zec)Nrc*FW|mGViC@At5|P!F?Zg7}CvP=E_P&?5 z#dmkVbLlMIM3*+h%MnoXl{RSUX!IS<&Rop<(z|J|Y7k%C)BB+{VX&}EdZZ7l7$AXQ zw5C~l1|&>BQU4D-0F<{pQkXl|#_|Ubx)V?fEF?pL#e^?83~!A1@9oo(4}0Irb};hk zCdi-U%VT|FLLfgf@O6g=F05m4^GQvbU6bs_JMb}0NcoAxPJuRZJcT50L&rMAEIFzx zTM+o8Jgd(lEtxRFEmpaEHAik4*`$|5HpbepT_2u4(Dyl30OAHhL{ob~o$y5-8 zn4a-m9Sxpx!Xrng*|!EMuI(XQ0=lxCWQ1cOwKQuVPsq+o$df)fVgJNINN;MXQtxe0 z5#lERVDBQ0_?6a3O>T7=>NMg~lGy)oP8Gr%$hpfOhvf#Xz8g4Yc*b9m&&(~gpb2;+ zfChC1fs-eohG8~|gh(^qrgTQa$Ye6Z9Uv^r+6_{#pWjPOLDY~w%Sk2O-35M^96v1; z;7B!86kt?Ph!32I3)5Ta@9#IpE+DNbZ_tLKJ%x+^Qp)|(+!e>whR)v*iM`Dm|CEG{ zO%FZ%{(hz4T?mF-CYRvtOJvmN3W?~M7_nSDRHH?Bu7nxH+DIBycdFPHOSb~oV*27=U}t?-H|-lqOd884!d#wL*)5-s|6YIU-7 zLLHBWAFXCSPBLSbMw^=_Jbyk@ib)&cE_4T)PAy;~D@=(vkoc04=$_f$yw=U(M~Gb7 z!F>UyWLXyR)wD$3+0!hn>r#2Q-bC)Pa>`lmBhW=_}%sbuJ~yC`Kw z*-pz1!>fx=8L;8M3&E%`BZD{q0;N|;W)mXlPVwBKJ;Cu<=%8g@5=OVIo&|1XkQrb^ zeGWo7K@);S8?;UT5{o#qnBhYQMqRnuc~k9aDC)ZPD|J;>Qyb1o|C-1{Kp4Uh=93cx<_;d)sYPPI|1g*ZBON7HsQ8-7%+O>OK(_ zKyJC)I^87StE;u-U3uUf2AA& zsyg6}BZUE_uxJf_43fI?gs2IuHX+r{$H_eSJMjb-J@RKD?T)|c5a*spggFF6;>_XY zY2=a`wff(2J*;okm$hy*H(%2YL5>vZ`#%rT2~GX^LJo5ZV46;2j@t z?{ywIGfK$uKlj0N!VdC~zCs5x4jQO-w=TIt(!E$17Mh$)J4o%&)64MOm3*^}t*j~W zx?AY|mzcO^wmSr9dd{->H3zmg=TgO)+Sk*ej*_h6%(%CR{NaNso@#~IyrV^OsJnnF z5gv{lmjVkg}j*4xoRpmrO42C9$406paQ81 z(9lS_gmAP9?amtif_({@ksA91Gh0pKnLvbbXyy+{QT^$T+-(7)Q{8E#D_NTmZ_ja8 z31Yp?9YiP=Ma@kVtE$gdph0%S+0Vl1R>u~32B(=XbSW8ZoKs2YV%b1Iz0S-e#>lN?l%%AA~>9Rj?(7t>n;P0Hq2 zMDjxf=4*Xb5Ur9UH81z|3)Y)+u^3)_PQ6vEpNXRZIkQAS6ak|5#$0)k~V+rQj88;o#q$w@}bRs;VI_h+OiDsCfBk^ z7}PVSWtHCl2}y{G^v{mHCGz3%vqXqHi`GQzF7)FfYnc{&;pYzb#}*#bm!xMgiWbqN z@-n8fNu}lz;^+1UrS1fmZIy6vqwFG+4~bpwD(>}9S;X>S zgwb$7y0|096HC!NrjS1tAr})p3FU7?{LWKliB}Tr@GOWkq)$^}bmn~47>fqC1&fwR zMYzxUd?nc_{gdrJb8|vi+uix_xv8;9_rq7?gtj*8xvXLCKC6&=nYBpzevZWlX7yYa z(3*2{_I;HdjOV5)H4iF9f;KycS~Xtoe#>;?sA8Zd$?9Jp?L}gil%yG>r0ngAkQwdc z2rQjWiU!_y69j!gSX(Qf#aY0;dC&fb@r0M`m-WSc=JCo8P$bR2Tv}Rcj0Xg5MJ1)e z@Vwe*_z{4TwQ;+Ix?e6B<%g=OAB9nsUv|gNo)L!VX<$I=Zft1yq1RZx_^jYzufPU7 z76@~1=)8tBD`A#i(ak5SPMbVtL(djSrQI<}e|Hq0|1l(QfPN_=LVbm3DHGY{%NdeE zO~Ol^A%{ZlZ`-e_F7K56RC5?=3DOo`A(VY_apa7RAu|KVz{Q?)4z4vA2aDp{wFBbt z1uVI1Dg1|lQ6>=UgVs!^%AUYQ#%q^iv8>iVG(3C1AZx0|M@(Duo#@*kW{BN5KN}NB z1xvTuEYv3oMrzevC>yjVPpZ7lD-cqfv*FLCK7kX^2%uX}hZ0k9sNz6;ax?@dSTw+3Tk-2R*(V|J=m1p-BDJ zYl9NA{VB`^mMw=q_p7{S^7MST7*H*pog-05`v6EB$F}%2>*Y=kzc4oCGNVx)1Zrh+ z7#$UPQw?i7b8rYZaiTjhe{rrJWMv?R(8kyMOIFkd3&xDPKN%^vt!2oPf%~8!f>e81 zM4HIUbQ6k%w!X0ec$PB4qJB#2>@Vh6^$PdyQIzx#w@#_Pl;FU@LPv%J$08lQe{^aC zgE`=*NJ0Y3Gt=Ru&;5Zz69llS0qoF^WH2&VX5%xdh4Qu|GA<+6-969m!)?hL)J7@j z{-`pODYBX`sO!OMsvQ}N+L?5#L^p7E_(2QsZ~UQdL&PG$ZZD7lh$3L;pfxxB`0?lJ zN?kAqjK$|0TnpqJpQ1u<(RQD*Q8&~tH>6s=X052141F1Bjy>9j+oCn7i^qMz>IIV9ckkW-A~paR668}eGW39sVQvn(x~M+J^MVY| z8%l-~7Nl9sG3cf#>nDV`fi;u5W{MrR^mRWO1)LPV2LDh9hvj&_jQkcDi zJ(<88N3Xio+3&pXwJ0m6UcI-VtPDewyZX=?FtXQwc7C2cpnbf~`~DY5%;Sw}N*e1i z(`#e#X};N7vveuYmRsLCa{dU{Io@GdV@bt8}LDRXr8@z{wMf_+ zw2nN!Lvsg1ttn5SNN}1>3_CRUT|2wE++g|D;JSIb7PYr0BeDdrTU;>z3!I%xI)1c2 z@O^N!$9z1*)5r%Z&d5kqqZCq5X%Y~mWMvs>YDyXQgSE(~*j%G(lVCPPS7NAw1}g;l zyMVWhX*ZiF?fZP~7mx4{cBI<-gNc%d>5Bdr*_IAIuSC6s5TwF%7>P-UpJxF}C~GU^ z7eFsz`7}eS#UYhtvBthjs$v>*ro~sQR@3}3@Ed+2eVlP!&1A@+u98x)-8L=6AMgzR zaSmJhJe_)D`vH9$P_u=-59vzHV2FA+M(Q@>G($_u{cUdPzHlrlf?s71TY7Crn!E>jl4}viABqGMV+w_6~E%+=$v-d`Ty%4J9$t zfDnVsRA#9aj-u@w09*{e+2{+qyOR!#Fbj0|+#2d%-2=H*N=js(KU>{ih&i@cBotO8 zAk62#5E1bQ>m5jD07nssu_jVvheVd=<_5qUWpYq{bL~uMPLPob@zl{FMBv|^)Y0Q6 zPXM$Z;M)qGRr6>`^rh2i+v9z|O?drZZh0FM)32{yp-)UZj>H{7OQ@i8lRBP|Kx`pY zwn-zwz>SpD5drJNIAUMcmV3yBO=#KVWYpD3aIJf>JNWW2N7X@c^5fqeb~57EpIr`D z0`o2pbA2y?23i3aV`^!=|0$B6lHviRq+nbDtOkRK)sp$Svz4;9C?QY&USC6CVN(2) z=zk=s(E!MS4aX_dkZ5|03jXBl8~^yLAmMTOg!S<{DJfq#Ud=39-jXFH1(`T10^{oP zkKjT)D?Rf2>dH#LL8`9}!=nO~wZ&4e*%i>Bbzm0(>Tk#%VA2A&)1-JxPL3n{nv2C3 zQ*P&AgpLYTAVF1wfi(#}cA6Ju@{=jEeOx9$bNlyX$dFd&1p$aBVzRwI&~tH#iQRo| zSO%3W;Eb`?)h~|?l40WnZEo4#rAp$?Y#K!nkv zx#(qbwGL;Cbx4pIV85<{jc(i!+Z$ck!i$))o#Qe?K7HbQ+N%p-T|!$bV3j@%X3So9 zAT|h4AXLN;8lUa?%m6Zi{tL-k!$f2_#45ITeDMyOCRq6^Vmuh6L$h35Ng!(KB=x9d@xbOpNtg33DkSUYj3^s3DO=rwlA_IK56Uc_rzb6j=_rKm!;raI$ zUTQEoWOPILKNOg%c10#7C8?L?6b30Mpds<&TC1p8Thp(arg8*19PSg~zsx0om3%YV zTWN5$0c-R+&be*l%d1BceRGjS}Rsw zdE?@!7yV$S!hYwzFpmu^Wl4x}Z0vA`bPKQlfEVgpo*8Skk*aY#CQ`cg4goBC9TO81 zAWi|o>de70pcZBaDWF?}o6{EDst;gTMC3-61?^{j6f?o%QV9$79G3x42Shh;`LN-97zXu#?8;UdAJTe_ znk!w*$R%<8-LWlc;+F$_4|RQm1~I$*8LtOvYA!gZ{v1)fk05}ZePi!212+cHnE>K9 zlSiJ_1}N};zR%yQt>K0w;3uR|gcJ%o&u?yT%S#(oQxOxF)>8tb?JH1*1LqMy1_Jge zmXwRuDWL08O~E513=^3{p#V~z-&a>=+~!zAB0=Ch1~xQ9B8f55z!W28KxBEAy-V?+ zo?EHr5Fa)Tr0PtWP;|mG)&p4uh2WdLGO>B#PXG*mpOy9LLF&&TesoJ<)5HQBsHN$vN^k?RO zoBsV4a8wG@*=TP6CLqLBe5U>x(Pe8@dT~MbGcho*56%nE#RM0LvNHR7e*{Q#Yb(_h z+iXu#LIRLwxOAF+T|E{>Jbmk-fQIk|p;IG2G~jA|55_X5C4Pk;m->W)f#IR#9NgNg zz8p)rSe#*<@VPlf!+zka@GdND{0J0efQN2xmU0bCP|CKQg*@52^hbo^;wF77f5&b? z9ls}v>x=UlTm?QWZ~v5Yt@BC;gox2{&1D8nfBA&>1ywVZ7}|1cbMaxWPYC66cl#XG z!d!#}zJ+pVZ|BwpBchV=-RHt(k`qL-7#HbJAr6;qv<~wm8?=rDnng(d_cZ|#H@lKX zeNC%)*Cb!HFv5}E<74(0sOP!cHe9R3qaPolViJ z-K;Kz<~mQ%W^EVShIrU*!!BkZx11e&l%iL7SvVg-dt>4sa<{Y+!5;tD@W@9CI6)E6 zf|$q`0t3$7laqX`tTC&=Fo@JGUOE9XPlekVlT%ILHLunOf!Bj*Moq2o*LaF@2F2E1(b^l26`j6sb1|UCd6{6GvfFo$4KmfRIFwDh; zkjYY$O73kR*4E|a7%G%rZa!la!-L|<&EJ@eK?5fQxQ96=ty(|I3xB_umox=sS19%G z*h|hX2k z2~mC`H$}0ZtY%rD|7>X@zr!eHP2z}&SBYw6;&u^^yjmKEi z`drv?YZ+bR1V2)rF&P<|1r&&)r0R2WC>nvG%)5t@rIK_2H05Mx2flslV1uj*R04^H ze^uaJn7Qdnvd!#=YfG}dq^*`(P0>%W_%e(6q)!dcQ(IDx!tG+%t!I={QTw)UbY zhLD)1fs#uqb=CAPoolEC7@lcF+FjOn(q2E6VZuv39C8fFxO7K{e-z`q*LJMt!ud>1 zHi)$Oq4njk4J6~`V7|CUUTJT2zL0!23VMdPegOBRfB_lgtIN^49N&9DzIC1b=|Pm#<8p>Y0y^z@iMDiWAlkSy-ZH>#wh;DLsZQ*%Eg?!T4@ zysj$S`2l#EJUrbg<|L-qFV`y)1EiNVH-AJB7|W&ghoF$_Wdh}qB`t&CZWtOaFi$b9 zn7+woDn~#-0QxfWggzR+Yvwy~Dk;e`!h@f~!`+E28ujOyFuC+`=*KS`K&&Vv`oXK^ za$d{PVPrvu4YnATQ6FA#Uu(;3)0$vE1?sVLdPWwom5x7ZuIQgz z(*>V&SP(ecpI0Y)MAIN$OwVeZh>d8fC@Mmtq`~hAZMHWy2x~2dL!CbZvj$+MsWsznlmZUIJWD0|RDPt|JoGUf zOq{P7CiU?-rvOFhXU|ZOP$=912=mIrRYR&IRM-q6)9JS=9MAOdV0Z20NU%sr$gq5Q z&TGxKTsMJubil^VBWld=gpZ?P+Fl}O!O}=vTpktm%kIl^rsKNl1s#LT>9>Xk?sWo_ z6ge`a(@nF%4Cz`Z*&|}7xHw0WoSnmE^;DPf1RCI>Saa2e%|M;$KUBCaNnP=CZtNor zl@Lb1Mo>Lmw?cKw8$HYdzNp1aUvqlsJ{H{YJtZZbTUc;vSQ-4r*$vo;0I~|%V+;yd zK7n!*yVgGCr;)gGv!u)`P#KGVWH@TMuHQKB6Hhd@ChlAZ8&6zJjD|R z7RS?(x^PDhCS_-|y`K%$V}s!plOLg|rpbR`ZHS6W?|S(rj97bCU-bZSoR?G!wNvwi zO%&X+lq};kWG^vWi%LRuu40YkQ7%j#R8{+5it^E@C`VQ!1rBFSFF?`66>|2?i^gf` zBpin(^-lmO?MctVtix}VeC9f}5VAcOm1QV-GoA!O01oel$q4ycg6@4ym%>T{qktYL zF|cu%N$qI<#k)+Z`Af-%>y3ZL3VK0a?f?qxrIPOS-ZiV@b);lU0Gkwf2Fuf_l|WBKi$rGF~Vg2%rF?bqvAZCiFJx4-X=k8Jxzaf~>3i0dW7@0f z+1cc`kKghoI1uGlz>RnW$8|~+zY}slKBAPkgU|Z>49d-)?t;8ePk(X1^+q*5%lVxb z4_SSBH~%m>1omaaLy7)j{f%r)7-+kn<(+(cMfIxfo5jB)8p~48NLTltl9%cE^XJ^$ zrzeKqv|@WfcN;Vt<$w0d3eHCaeC96CwDOTd>k<$V!@-txC?9!cWR#tkcZ7%hA?q66 zZ~uDmMkdK*UL0PO^W%44B2}^X0q*Z%^9O>-rl!DxC#Y*v_(c5Gctzz(?zG`=6008o zwH_!?(!5MKm8K3Z!F=2^o-sys0&qEBF^qXfcUzNRU*8;5MumspZAd;anDfn~=z2;Re_8Bh32-)f85Y#AsBNmF)l}LX98xi{N0(j6M^7(HtfJacW)=`}1G_ zehoa_C}|I_fokT9?(!eL)C+z8-hmZ5&}r%E*QNYnR*tFE{!O^0l9#2d&Iso-v_zN; z95GNa1o~Ux-GI|+aC8(2grvWJXPO1>0wP`9%-GW1>oNb6)<4^W4vh`^>;jrm`MJ4D zX>}GrGlUA*nZQ8~SVH~REHtbz#-w~)@zDK-^8AS8!^ceIe>6d#&oTqq`~m`z9$1-M ztXLd-wG~btiHwKuUSjDpJ36~}4LmBOc+AKNZxn4du>$iqqKCgWHT+C%Kqo8-L@z+8 z_&4B6b;L(v{u?Jx*NDzCa@{Fo-L?swhOO9O?$TtUxu(N{n1VtDZxZe%$@k`TU-#DZ zo4)R)#q{TXUiT$}@vtk3;u5a05|qGFfZ81Wx}zi}wtL_1kROi!b67=o19+Oge`65$ z>ER)0NaWkJ;|ua9|24^1SzeUS?tih$om6f%vpx+-uYYOl86qjmj0{FRj$9y;)zwu) zLxYBfW~eElvb3Op1{3t6JL~Hnn@ToIAeGhM*B3_q`~>vV-!~q=+YW0UJEy#UGOf7_ zEEf(A;@U~n)|@*>pn(hZ&U3co8Fay?kFg)t_iU9%F}L=t3N&$??lvb+870P_UN!7Fd~JgH-oOP`97VztA=ivKZs{ z19|V;Bl4p0T>`=X1PZUnBr=Kb?)oMsqYkf-Yier#GXgw@F*bQbaN3<5u3F*(ubgJ! z{Q%^>tOr8ABAlGZz;AwQYYVV*ylbDiH|jeK>T#YBJu{`It&a-$yYYq~r0);5lrYM_FYE6v&qCbcWDc&2S?HNU>p-L>V87EG-qPDzRhBQFl9-o5-%RKV}8pp{VM- z)A>m`$T(miCKWHQ{>G#I-d_32L+;OPR~}Mj{>@&#SGkQFZ13L{f%mRoCUgYR$L ze|W8Z`urKln2i>(Cq(jo>H1p+VOkrq{sW?cEG{M~sqD)}mQyG=+77OFL#kwgYZa1f zm(82f>Ap;{N1^V<#$tjI*`9Q0B{_^(2+ZU;73G6n|gDPXM$cLMbkC5fv|8HrKc2d#)r41v@c#Nz;~-96UWGEG#S# znbqcBORn~sUjY-uR2Z{3V%t0EfAYgIa(5^sN3XHy@Jw z{+QJ%9BLxQeJuKjr8CY^7GXn5ss?sb4Uyx`w`GSI2>i!JcX63=+Lo=$fiug?^eMmw za|29$GOyY1?kH)Ul_=CRKXL{5QKz?>TD56ur(jHGx&2Sp_f#IX!ignkbUo z@(<|cK#B}Xd6R7GQ}kuy-`VzM12Dj|{K4c!YvxSOECotQ+%}#rZPmLLAVHs~V9pn8 zgM;yJ60!eU3Aye%?j%>F=@UD$v#XB7$w%`^+wY9O$OS^ije#NDiDFaU?ozLQ@NJI( zyh)y+?N>owHs`aE)_^}i+8zX4>K?Y$FyLwa*{itb^f%Pb3rqr%U+0Hf@;FGMde#K*}*P-rEVp3qwS;)T^z!E>LP%-1^vC}Z*a z%Rv+0Ak-J>PfsJ(IYeTSXAEj83NakM)JJ6y@iST@u*{4jiJ)lkH;0|3{SQr76%bV$ zZD)X?yBnlKx_gk44rvex1*8N-I);)4k&tEp=@ulVQ>784I~Al;;(q@3-e(_>!#O+F zULlj-g~B8EaMIt{sbFKRPrC-(gen#IHvzNXu$sPc)VsfdpzCPl?ce*^U;f+SM<}NY zb8`;4qcRJ=9^2=~2`A@efBv|9`v+Pw{fA;HeV4@VY%=2XUI)Z!3BE#6@C}KBjWy9& z5_jt7&({Ckih;p1IPE&jxi~tOXV*_)i~RomJ9nl^@m#(yx%o8Np~1q$M711uABJSd z5$e9U?cF=feEH%1S4Fk>54taNpmPIuOlI^P>E0A2?5hOxohtlQH_^c&yRGou>R=CV zC6IHk zNxH=QFtqw@l9!f;XH7#OWQ}JQ11UXMPmfGt=R~b7C@!{VI2(UBX3oZ353%=I~E%&wFcaxKK02~>R;Lh6n z2pI-8r?s`U0F$np-=_cLC2fp?N@iqKm=u)ig$4{wz=<&Z4egME$pD3&>RO|wnnH-g z$J5X7%@}J7j~xg^MpSfbbCZmUDhe;Os|HkC8MeqYVx(@@@GuY9=ehn~UfOTWVH*qs zkIaxk-L9WM#n3ArXuMZtJP=KCBOq<}zdH6z+M_@YKM8xZ=}bU0?dDq@l*Q@NF)^Wc zzVi-(_bVl48fpfS!Wss{9MHu+qOVy(xC3mxWZ+4g zG>e@N8k_`jN&nEC;vGh>Qf<3H$MhqL(9+yXM(vN=M{&ly)o+a72U)d1I4ESmfFO^+$4Xp!@4iLp z@cmMnQp|X)&t!ChF|o-p&C{a96+K2&_L#3W3;i!#Q{wR!%j14*GVzG@_w*ZUAtb~$ z5i=|~{b8=ST5PL&v?T%(#Lr)6s;`Aa`d1kfVp2Y^*tWpLZSDL%aGffq zKWu%v{3Q9yG5*_MeJ`F1s}tb=nz!!Ohw-vKTw~|O$GLvM!BC#(80PqI&&)m%}7AKSGiOsOSAdt>cI%8Ad&O1k)Y1gqT!;e9tQlI9hLL_ zY=P4qx%<()mV5fyhWkT;xS;!spQ2ymAsJ8skjmEzUUKq)4!e5QYTPmk=e&;5eGf_b zQLf+Ox$kzquX1*1DG*a^b3XVE$BO$dX*157>i8tUI(5kdLbqPT!gBhmBQ5y$|JO-)q$pV|r*KUpU4jKC^VSZ@f# zAhPDREH87j4*90|b%vINQFlwrMxxV^UUtxieD+oD{xU*~{;M6Ss}O6rhT5eirwzxq z0aW7CD0dyqza_3;InY+(({Xz(9S^zvB2IEz%!MOsK0Y_#5y$6<+n=5xSF(EjBStwN zei*=rBYa^{gLOa@kB*Px$Di4tq&|be#y)^MW{k%ph8NfUZGomsj6pdmB|sh1xj(9w z%3L2BdBkxmgxw8?Emr>qCX^}}p+7E42 zY$P%U@~E1kK&qM(47X2%H2=JBaLk^}5MEbH+J~|*9(YQ~znm6>_7rmnn#QFr!KOoG zI;e1If{&p|lZ0u-$sm>-yHy*P<%I`}zYH_(rI)}vxNxN$_l_5^mVVgp+{{|?=yTY` zBRb(paiyVN{OM`m?ezp9i6JMfJnqePH}VHD>mT)UT~SAXDT;9UAw_v@ zP5vd;Pew^D@gF-e`1s$BX1akt3i^Lf3zQfTptxGG4z{!#;7|u-NY74BPkZmw?CON8 zPQ@yGEq80#f_L{($98hBwip*=yLbIcGo&S5r_m30Tw51`R7r`z0C%Db27f-uhp4C* z{&C8IkjR~hVj7>b_WSakj1IV7>&EIq95y=q+VqgmJOzvtUNn7sr6cip`Kfm=UD+|{|-2|&^wSpU* zc_`8=Y^Gq{$(Ar79&{z(CI7gjb;ZGjP0YSLMVjgn7G`_|o}sAn24U1ABh9#3J2rpR zU|C5a>pVxg>Kc<*vpdr)b9#D#6XMmW4f<$xK(^9$cl{d>^G0R=?g@=bpL~2w+E)_+ z#@~@)Ur*}#qzQDuD+B)>v%^QHcE54kW zg)h44{pc5~kf$ZAe%fmI^l8B5p;Z>uahUOwq$E=(ZB_HowN1_mY1C$MDedPdyfi9= z%KVXsGqpCTsHnBewXI1T-9+uz*M^(lh@8V}AR=N4Kl{a0Wc=!{SM)<*v zcYLwHVe2n|T8?`N(H_n4YDcE_B+JKi`VP^sT-28kT6^6iYaK1n=9w4U!Ke`4i z_;0aH)##DSw<*WyurXlr`XB*#ZnQhQ99iPvlSxZ!)PJ+=(v1^%YW!Gea$+#($L>cF zfj9|TOsZ|Vz%^9mF6}W@Eqq`XLhGNo#*`0FVSS=maQ@2yCF9hC53jLdLi-0WS4I`S zM%ZK7=5GvXw6PlZ@>r$y>-pDYgngJx&B(~l3rqBwUiCL?hP`)vFWxg$=Xk+!?k6&> z7^UA$!P{6#c4Nh^u7tE(m2Z)~U+lP@@5mN+d-T}Fad)KnVEJlo;(X=<;POTE5EV~K ziIVRCcjGbsR;PkWh&_S!kC4n+Rq7R-xg4-}Wo2n-XgtTy$<1w?b&3mq7K{P4M+%90 zHJi_4BhQLG7z&rsRF|`H>#-AF#K%ZDF#i}s`t7YcscB6Y1iY|I+W*+TlsQUHlI3Y; zf`qa*H=jCn_CV~QB_Oz}>871HHh;q7?_BYeuqc{B0h&&A-y>y_thyDJx3A!$xuL_u zOHMPatSzj1CP-b8&g^mQOhu|=jry12b*!K4J+f3)u(oqvd1kJ(xUQlYhZVd1F0Za0 za3=JU^KWcu;8ns^{808;du}s#QI|h1lg!1f>36>f&vzio`&VG?>|m;msaxhF80-v7 z^tdvTlBQHi(#A)q_6x1h;iowm>HztQ8xX>eIj3AI0%ImL5*&RVlB?fGgZo$=RBaWt z{!_1$0%BU4o{nMaNeiPt1}5Le2?d*w$7q5id~E`l8X6u84nk?zP{dmLth5`p)!(_!!p_b)HXqZk2`sJylvAp zGEFZzoWLr-(zzs^=|s`I^}6Hkta>z8j!(LU?9AEkS0g+996j#s=>5%TZqVJ;7R;W& zN1uY6d}MTVz2VR4m&}T@!6tWrwPzN3QR$P4Czul=3{^WC+2M7dpTx38szhv2S$hM* zxLwmhs91U@xsAZI@jLRYeT(t#c4hgrbw%k+7VSPhR|lu85-ir-P8`iG3M#!2DK z;nbw0b-}PM8d}=kI4^#|*K1O(_0>J})Uz`seZv*Zb7?tNhK3I#XqG@E6Ua3j8a308 zdD>e4fjD@49+3y2=S{CAA3LDHh2VPuOE1HPS+Y6mI3{4p?9vFJ8#`+22*9e3jYl8q zUz(kLiM9i*PqSfTEqTaBMMf4PH!vhB@ewZ1{3L^_?sVeUGMbt7x$u%pcu z+MqH*G@h+jQK5$#5=W*Vq?rN)$(5S9>ABiPE{AcFs&4s55^0(0F~;`ASjgbTX!f@e ztdd=xMuKef2wW~rwwQldP{r%g; zgFtqc7NyO*j?KBFuvhc#b$I>~AY5{^C}J2kwz3YaN69o$cYq)6bOQ=A+>G%ca$uaG zQd?8=MfHmzr*r8be!R3Q(dJU>FlQ1Hl8A^1a7a0H(ixeQ!a?R?sFP-|Kj%{Vg}E}m z>e&i-q5rspJS{QaqAV=$0^+TNB}~M|p(3B^b`#%Xh{98kD2j0*6x`YxJIwcR{4_{1 z8isvVthcl!UGiIEQh0G;q7t4^rJ*3nCzUr=58Q|iZ4muJX=WiW|7+)oaktv7R=*d2`Sah=G{4xV~?fNjzTpaj|VaQrXJK z`1RjOR1)!LX9w`1zkdI2)_hhETpmlMr2cHde^pdG9IBsnV&+z`h4iLj#rEF-59t+~ zXI$LeNzz<=d;=ubGbG_;efZU3Hn0o~HxiiCpKVj`fC-bIPF&Z@QaDejYy@i`^t|XPjzK)PCdZ&<^Xu^;{u5oBXAOuzBkY4Blb%XV0cV{J>#vbN6|FiL1 z2u+o|o2_k8e*UAAFbTn0!B$rfB}CxGwtWTccMN??+ZQX=VtoG*TL(V&_YtRXti)3X zZV^dTqK@AKun=`+iR{(K)G^({Bs8e3yv9!^sLbutLfH{+Y(;L;N2}>grn>ll*o{dx z*S9(^$Te2j|M2^h(fA$gpxT%X=?9RIhL|>?8Q`dO`d59wm<^Zc_rfvH&gkT&`E8&Q zn(2&Y@6v2a)wd?wBZ^Y>qrSt9_>XsU^K%alUi`md_fyq@;k;lioTgJLX*=~Iki7fb zJs4w_k@Fxc0*t3eO>pb@QGU4x&B#o&EtNbbw zGFB_-5#>1|G5M2Xi49?P_6g4$jTh^~V^@iT<&In)3M?!vaM%X^R!zg?xhBiL{+Xzm%hj;N_07U@a{Am7}#&O6^=8%{-gJEtMH>V4Cq~3Hg@Ek_eU6 z?4lOMP+>I=l#Z#%0R<)yptz_o{p}v zv=pR72}lY@;Ew^HsQo*Z(UV>sG_>sOT{+&#@Et*5e6Ev_;g50%b$O{SUUv5BNHEW`aZ-Shw|ME-9R^xtvqf_MnW)%rS(}^)M(4o> z&(2y&?i#)uQiD(dVeFTmxvT?(skVB^Jaxi5aqi0Sw#)c;41eBuZy(Qs=U}0st^LoW zdgcoRual39>zA?2DYBzOuGc8qa?>>E{k8``Pt(r+uU*{z$q{%th~B-#7bT8T91mq= zw7o5a%o*WFFcWis_mJ~(yd;y{{!n(yE7J1yt>bsJs1UZU_;2CNm~d_}$A$jR4<)25 zc)qA}A>z&7SKP;$lAfW3RyceFB(xb7#)s4L+p`qN?JfKGqb0137x!;|R6VatSD{6R z+S|14s0GX7U>m;a(Fq?9N8L^nur5NeKcCs^ioZlk1?T?GDy5mqg9Z+~;9ReVoSu(` z#xpEjo!(ZqQ^^0xq9GW|$4hd2r=cj*T=YQp4%hNLQ|#3@C`nX41qYOo(r}1|&_b6; zfCN!p%>GE_3hZo|H=h<7VbA8=sZ1IbUA_$Xt)I0Vu{=3O_WBnLg>)<@ATN$Sv1OkiXE&*Pm=&tUdB zHm9~*VFk)UHHirdQ2X_ay~;e(7jtg(O2gx(HN|X*Ew1R8Gx0|^QH&+(xCW+b+oG)* zE815dg7P{HAijP()j?&gL95#+V<=v7(Ac^%9*9sqZ8K$A?i zVIx-J?0jZyb52b?xG?iWQD0bkFTjf-73>b+7c+eP!^FTK<^B8jSy|?$rjo=|?}H|^ zY|u=a{8qL-I_{d+aW@{22V=zK+x1nap|geH2&m|CUW#*m#Tj#WF{FAF`(|(NhZT|I z+T#$x@i7AM#k%h}$A%|^mB9`(#Cs95Io-MQS| zfFtkEEXa=NVc6aIR7AtI_#>#ju1+nWBl`kC0J!HfoIH+CeN0-|sbKYjc=eKJg+-La z$UupB8sw2GtZV_^?aj>%&_L|%?Ex>vEBl?Oo+-}YH1^i|bfSRkoS;z%{!hwaWfiQN z&L+5?zYR6z@E@w3I7?~`FhEs2q~?!E7%pP=lMi<38(^1vhM~jAb}aV?{rjuc>cJ)1kay%z(sopzLE56hJT0cexOTXUDwu^n9)! z4{Fv+D3C*2uCz=VmYVtz3cW@4Q{k@SJ7Z*I|UQ8!UJhr+@$`o^a+?0fhdvbeNF3V@+c-U_<#f6tAI`_0{itt93tGO zmF49$3=9|op*cx_v;AK#5j72sA8=1$V`G}d8#_r+J{UdmnJfQF+V&^OlUja|L>A?i zhVDVPo(e`VCqL;-@kAubfEzy#d7D|c3VPbiwDLyu71B=hx1iwhrTUoA1>NDW+uBry z@o*YFoK?@N19Pm{?REbP&t}6YeI3l`N9FtsL0Qr7U*cwbtD0ehMoM@~r@Bw8R((MI zn)$1bsMD=egH|u)F$^t%ZdJrBEB)AQ3(_jUU>`qRD_b2}_%=D2#wkKMVzuK|Y(MD! z)&AA?=lin`d~L00n6K0hSP12IE`uSU0N{^>vhJ@N9`8_=ePpZ-7Py|Z%LQ?RL*3mW zv_gp-D1{%xx_F&j4f-V-Mj;`yUL>$sJGC z0%SDk>0WJ*&CfvGC9`ket0=NyKa*zS|G`_y{*pC7* zI<7Sa?B|o?V;}%z!h+!;azxGVep!rXYD;9M0?X%fsD1p~TC%C1+jzN+B)`ZIu$Oi5gZ_PRW^=B;xkf z`F-@6&=1ZO5)=e?jG}xjl$((%1IX~wGX_$}^M}9u)?fE)r^Y8IA3nN_@t}A0!tW)y{(Y^X zBIC;RbfJav(vn*nu*%1zA;%|ewKE~*)d}X}$|{_x={-bNOk@xbr1+oc;vViJkA5!1 z-N$U)(H?^qqB&hfwqM}}dCEma^_YV6{bu%4GsZJ!dFnk^nunFd*BJR$38ah0~{Bo5MK<3!0vYca{;J5bSInVdCvmCthAgvZ%P205InE>P9VEJvH&u9zWMpiOAWu))WzY zVu}YMsx>?t$h&6bU}G+gtql#Hqd795up5?(fIq&KQm9q%)e$lQfuSxOfB1WKhQ9vF zN5)Ph;<$61zopBv==!Upbp+{oo)<)P^XgmLFZY+byQEWS>a~NES@V&xGlXWVp;#Rd zx{s`C#u2Z5%LB1reGlj6Zki+iy`HdD89V(gx-88%?&5ShlmLMjui$)T;R4lC+YfeY@3U=K0#)ob+8m#zK4v zpG0k++h@Hb8g#5nY@@n;mrN?d$B2)m)@ew~G`1tvbyX{(4Ss^54*ZVYIDHvC4MI&A zrBsI6tx6Jy67>m3YA!dc!j7uF9U8dI@jQS`O(=|_l2%;)pLZMrOtOTig`eUImj7&2ZqMc68GqMo*MSEGdJ$ptdNt2Nltn? zrbNN-Q1M3z9Yg1Xm|t>DfTc5F?8zI>e0DUFp=nVVUsRP@N9x9$2s7l)+a%k-8pHVv zeSAVOna}_9#iMrUQy)&~Ed~-e=sRIlI5~Yp#R+k8p0b<%G3Kvqkpg+oQz4CaX+weE z0g3a~&9L@3v{sE`02tX-kAc%G#3#-XM*K?O)v@WKP&QG;{hM6(c2yF%4ay?AtXb#S z-boK+Sk!y3s4qWMkgW+kn^2MBLEG+rro&n+q??|V=4Mxjp&NnX=iu^_K5J%px$n61 z*n|AskG#wnTg?Rwy0@s_+3LP`ER&I|kXRw{`pErKw$s)JA^qv$;QX|lx zg}@9%)62NBr*zmL6v9zUFbL@$<_xTGmE$M_B7TF$8G@qeKa@k<1SE6VUi99r{n{KodA-Wz`>QAx#} zSCls8mho7~h=|fsuFrV!^lA!!dr<7BocUc|{Ni5oP|}9r=|nsQ<>}Y|_VBJ}F2n!I ze~bqC7Ctrw-H!s)Q*#rOE6_y0f?^yU9U1$Rl#zxK4S^adEJi{jA_4*o^SB3-EMLCl z1mIT!lt&QJE~g?C8c!XIj)aCY+HQ%SS*_}b)AoKqJbuw6Qi)|7hl_L@s+B0~XL@HX zFAqn&Jrsi`(@@)sJj+Y$>kIqK`0_ZhTN_;e?XA^$N>{+c8%3m>n3Q``{T5QC zxqb`?&JsmC;or_AJP?NJwp(~42$w$SYW`>p=pRIHdxaW%-R|V#Q^Ye2M zkOVLlpcVm#IxxM)>$v8;8Q5hXXXL#a-J9!kKMri4*ueogst=kwugR_nR8^M*UTYQq z#Ku9OrlBy*cG*}z5-Z>;vysJzAyd``%{;q4tjLYkxi9ux5ul7g6zkE3e*OJ<$RRUD zwcSZq5%JT5`LkGk321 z>EpQWkf1JkK|DSlh}*oWn89kQd1suepg=iJ5)r=0ZY*t>z*CyG6-tYVl=l`-OG95l zQTFqV9O=qPLr3JdhG2DaU=Khs5I zD%TNV7C%QBOJK`OLcIKwWJy$}j8iMv?&HLo5#08K)w1Y-@e!>mEraL$;MZ~WmDB^f%4T^9;ywDVe~V%cVBIAi%u62LWw7+}yvx-N&o2sp^P70I00~MLV(Qc&{~Y z=d^n4zF(da4gCMz;8ra;f$B2qF<2WCD-B%^oXIy8i(7`=4J{vJ2 z{R{up#G?IMC`?YiWQq5NR-R_M7lY&ro8*Ij9paTo4cs>EQ)N#q6;R3Ib1N&u-tXnb zE2Vu^3f7E?Ijg(@J~W|Rxt32bkY#Fo7#gF&^0D(}H=GV3Q0mV%Bn1XE>Rr8QJzWAn z+#pOR8vf<9G8}OVt}8=FnYYh94do2&X-rGrJ}4VOFFTA&s*LV#H2oz8!_{)fpA2RW zHC)x8x8QDAorc-BF5cZ91Tj3FJP5pK1y1oh%n8y!8XcLtGFMt!DsB+%c%>qDO=NT( z%5Z}KAW87JLw31QHOkIU=%zxVA97;x883u~7-oP;Pxt-%ccAm(Yro8=yI<{`_Eg(H#pOqv*IBpR}}} zGyE~)k161$Hqi2qkLc>@mDoTFqO2ONMVxd5T!9hCVNLaU7!EjYXoE1CZ{O53HQ$6& zmH_yqgMNC?!~*B^6Zlppwbr>h+r>9$=5ZYyn3%acm{CDTbNG8$L{xbqo`@`~=-o1yPztOjJ@>_`=c>0rbR^z%z2e!Nu{7QlW@*`k2*~%m?Pd7@!9_;ZLt z4#Kg7mL>uO7HQuSqqg)>FVGs;*$}J_CNl4FqXZoIE&Kf{E!L-Lf}ax72i>@x4(YO; zb}}pc&jQEAiMwZ z&7m9K1XIEb$zJ)Se7dSi%rVBfG0iWrgO+M=F5k-7AIoPuK$j?-;Ny+jyl{-jz34hzbK9fqpnp867mtx1WvzH*A z--{WV|L(Se8AVqdpr>LLPRYmR;o-2Tkq2{FT%1R5S#UninetR%QUyQ1h%7#W(4TBr zzjSu)3w55h5#QWcdXN$xbCF%6AgMyquLU-ybXZm&&|S}>W9)zzRZ4jV=4ideUHv70 zt0o|$O%^^?REX3Y^0~jgm=(qSu{9nY+lN?*kH;_2oZIN``r_#E9evIkzh%q;{~Q*# z@dkdFW;i&4@~MEp@}UmZimpiJ1erj=ZnYMU((oU zw5A->Q2y)pZd;Hm)FURQC%&1876~j4F7{^EJzH^baD=3HC4oKsVc2VN+6O6Ts#1Mr zLf>T6wMO?k+(eTWBmh8|-|Ff=!F&JNo(5Hzqy`H~Vu)j2zV%PYQZx*0@0U1e|I15& z*X!Zw+3?6<8eHfi7y@_Pn`7NXYXm7TT9G>a;DhhPaXuFHEjmB!((_y2ZXh@~}!_RxuvSsWE`2KvY zsFiPy*7^Xt*>O6do`w2|WlqcC)z6-MnC>$MrISsh(@hj^(ciy+4T#5pC=z5O2abt) zqr3-i!K^~U!+o5wi|xU1n+jF0)ytu(R@b4}cQ^yL%{vKGRZmalZ6wSyt~a)yg!C5{ zTF%GE`^GIZGcScbY%i-gMK|g zr66*~JhpBxI5Avb`#YW|*;3&MpELNkqP6?oiVaK#(fL|{vK6iG-DJVR=yk4Tu0=)sog|MYK?>c5czOLyR8*I{4z^S+k9b|LB!aAagh;MH(N^#v!>5<_28JHba6FnI0o!%K7{%@P zAIyaQeR-rPGNF0$6JDsYvmhe9FhLx8JRB4y;i4_f}}_jk__iHmP8Q zS;)FzfDhY!NO2yH<+!LO@1129?;M?G``5b3sPpntoW9--ZRfV8aAe+RL^th;*l&aCr2~r)mG3!RHmTHq>Mq!vZJ$FM?LY$O65ajHtk(Yr$>bC- z@hvR2e`9MRYeMfAKgVA$yJD@c?U1d|eT1Gib*ck52JjXx!UlvWEIxSdjUYT=OHy`GqN?8FG!<^iA(s(b6IXk_tvR za|sD0n0J9iLhTQi#`40ZcLLkMDmW=AiNNRiLh3BV5FS#j#kGXk-{*roi1V1(*#r%i zKnU$*3+$pIsrO)B z*Bqpz)A!Ok=v1OtO^&`s~i)4tPBVcT_mg0KD?XIHj>3V{@- z!LSFo|45e-zqKK$*(1@G4ptK&39FQrsp+;!kTJBd`18SQ+`EAT$f_vtyf;&f#5aVn z2PO#QZU_ryz$XCA>+B)aS2Zfe!90J6P)`?leugFy08Z5Ve})l<&bEgDWGl-Oc;8MVJ1f2q$sO)R)J&KB;v#hQtv=3ifJ|Uqs-Y zS65qjPo9XZ)5VAdBP!|WTHa*QRn%g7mw7_l@V9J*m3^WRCc2DiWE!}}c7leGIA%m4 z#C|gh$|$N#G9Y}8B8-bLeB>7b5gt)UE>(jEAzBq?o@DjR^zIO~qeAB>Jxs#8A%h`~ zUD^F_;HkKu*cSZnuC{#?-qnBQUI#&;NC#WD3RFF@>*z0F$SYEV{sAmRW!byg(ZIJ? z#u!R%VBRSiAr+>E0izI^(=i3}!GZ+YZ>K{jdas{LT6o-Cx(&%c7}{xv<$o>;PH6WV zb|=uktL)JDS2MB?S?HaSlyuA@T{4nLu$EwoMGkjpKN*i;B5{_2ilpJ7V!cDlP*rE1 z=oag^PJjbvfyKsG&uDqZUd`6WY$gj8isc$oJ``zoRs1`Mg$?LaEd=P~R&zTK@oP?S z#`fCYr4%MPMuJ-rVp4=lx%Wlf@OW+4w@YRA0Ze53zR|r@s~J*Q*gG&sJgrr)gbv}j z0P{Q$-4`+J@z<#ic~gLcp))TYLdYK{Ba%8h-SDf%{;hsuAe36D_Mo45?4{MDh$qyb z;Fj^FXzGD6LX@d`xt=XJe}&Vp@+)fZ&umHW!Uz5N_!g1RwEuK?6EmTklGe7Tw_iK| zz1VVT{x?vhMaxKP+3Bzyb^N&_T2coQMqZ!x$o8M}KG7u#_LZ9;i3WDrr(k4GqwWY~`MDknM+ z>Y=#kCguuraZ*leYMOjcPXi+^KK4+7YdQPaqhZunkwN4;vtuAr34>55vrKKa!FkS+ z(AiBOz1-kZK8Zm1+SHy$>IR9|4`*#6u=)U<{dTWdV7y|1Fyt<9sG|715?MMP=F~iZ zH9J?9n+l$U6Am0uR_8VjYYOUTPA0>@cYtF@)EaU(P;Hp-M}ZAsijZC`vO@>U?nDj)eF1_0{e^4+P7i`z@cT=M%jYNN{dwZvqYl z(tuKGjGo}9O|YU{GCX3X`W_!#pufI5}I%zLdLJf&7EP<6ozf`J{uF%Ow-@&Tc#u6RSfb;vg09a(Q z1pjYUi$QxRrT_*z!8y^F7*Fc~T{g9T;fu|a~DxK@vwgNWvrq{6Csw;GjePAI*yqC0TQylHHs3{7V49{29r{2 zkh%Ii31wwv{p06}gB&`K4;U!Y_xUWBFujV=a(70liglD3CG*tH02U)5bS`0mqI2%m1Et&3My|esbdx@>} z!?iD)Ht}4pI~qdcvpw1cbWb6YkvTzkJ{@;mQY!x~IQJs}lT>sg$r%HeE4tme;_NOX zrXK@(MNjcZCV>D-XGewDpGD8jI3ckSA%Ch<*Dd1=(p z*XHKhhfg4;^}7Fu!e4KdJO20o=%c4&e-i5F0aoqdv{r8gv}6pK#>!ZFQ&Us5$B!u& zoH!9mG{rgezNqb6l6ZKh4U1`|E|buDU5Nc>6i5qC6nmFHQAEC#%7*td$(<@BxyZv; zlVx>D7UsPT?6Q(AobQvsMN{K;_~`VRoHPfRBr0PT>gM2BJ|5yy)8-ezaUJYlznA*~ zBD%NdbVkCGotnDk*(yKrkG`|Nzed^CGqv2|>9LVWbi6QA7vU5(&OZsj7G8@lP4o~M zz4fQ7OQb&FR*5o}-`OH`wHo?l`Tlz)6?nt#+fL8;!!rY;OiWCib{^oy zjMxLS#KGtf`Ja?WM@J9lKI5YspZxkYVJmVM(z!}*oj+c)QoZ~!v`&UELz}3UR zun!2eZ_J}96BW%g>*~rsbzNbPImOaj=n-Ev)q=F|w$@guct~9NL8$fqrXu!2{O=IuGC8k%wx0uu0DY}U5)w#S! zh39^>bZquw0cHJVd=jrhl1oD36jd&tOtvz=aMkQOR`Qyk+5xnJ>Qu6+sL1dI>MUU? zy_*7RUx1M3oAFWIuH#2PH$MKMaEZ={Aiy`m4Rs1*UDw3?oAyCva|XP!x%EpnZ%N9jcS=I0sobhsq=5jKkz;{^o;c-|ZD zM;(K}>bF%JxxKjw^f4eS)s}#1b88FW^xr$$CX>@Bhq!G5bMjriN9eG7iA3`&ylU~|w|<@}@^d2gd%u;j#*mHPDyzmM+$kM+=P(%to3;0YYs)4NeZ6Hz z=0jfHS!&~8@9c+t|D4$R^(pLgW)`Y{9|t@*-{!7C10d+%WsJ`nFD|e_rKwG-3+?_^ z*7W8{!1HPB{3uaDCDc|_KaYhbw}OU-bI_5rieEqV>e~gG)Ov|~02p)JoR1S ze^U0vwFOyT5}&k2EdL;NXm(Qx3?$1fsfR9H*>dqxgqG4 zi>=zfA*`<;g$DpOSp{8qQmEqhJDXg6vuIVH2c1EdC;KrwFJfzU6>_|p`V?l!OM`6qR*(-+0YKbV?;-uTw?Cwdtw_DsAdtHi!3=5 zMhgm+D!Y6U*o}?;6N_GwQfup29{+QO@I-(_6Q$zcQ*?H|zT>hANJ%78x1otKg_wHW z%jFn1-I)C=I1bqwtQhvgLV9C^V+4=tx@%Q1WQ821euL2CN*Yx!4-a5B2P6fJzBx9a zvUR=~6JkS*kmN)Jfgvw2P&V{)7UsM$vXk(3DOd`T8TmRfHX-5X)KuR%N&u5mtGW7? z-NQB!wH$qhsexY{!Sl)YqVM9bv`=Eo(4tVRf{Pv4O2%y`<;hiIH5*oR`SUqRIEsI6 zktLJBI!Pl-zi7hSXv=A0VtHvxVE33!sl&gp!Q~JOp1v~9zZu+fqM}@)%5}>kD$wFu zlbBOVfsFM{AE}KJT(S}4p{MX5EkC5o#_k&iGj9$4%w~%Gp*@xs%Lvs73i)em*L4?i zip;jRuAREcg8QgPHZ;PIA-)m6|MWI*j;MembAHJiX?cdE-Z6jKgplddV(}sh1%R+F+19TAIiyzUm>fY@Ah^Jc{>s#$Uj23rM z0Rv7_J39W^@h5UicbMi;a}u zFL`Vbjm{y&BMR*eHJK9Gr$CN#L8_R%Kg5vdZJINBlE}GHTZ+_cayG%JHLV#u?p7cn zIpa>k?KSVVO2laT7ki;jX1DkWnZQrqx_lx^%3Q$G!C+8(dDb=y~ z@jhA{F6`1N&4r`LWiDhM+q^=TBrtdn{kuNRIQ<(qK^f`m)5`dIxVZ^@I=2NX6CVIT z^N`~dDY(@D_56BmsHIE!RM{4w;scrvrH^Sn4~$j-y-WxzvY_Jq4U|GuMClO^=x5?> zHq&(?yOUZiEGA83NV5Vqgf07eB-He&22xvxE}q7c^hcz6*e(ge{$^-|7;|f*BI70X z+sLJuBN%~qBF16f^CvHZAsNMWpGs7})Ym7&7n&;bJ%v-|OP{0Ivy!6pzvytj`Dv`L zUtcy9h2`)idd&LaH$YI}6kjq%@v}yn>FGUEm!By4QJ*+BO{XMil+ePiB6aRUes zd0sQF&#kS2Jj~~DbW~K(<6}q{AP)iB5X5cAM0d|qDsBD{jfW?i@nNccIp~|%It8Do zkL?0;vZ$CC$1hQYC9A60o}5ZrL~A{)`uzQKIUiwl54;IwoQtDZ-ttrL=Wum2mf^_K zw^(Z{hYyBFpKS&s)@}1;X#H0T*8Hl%_h%hG(zQyWLG3+(RPDnDQPKBik;IrOeqCRv zEhSm|e#aAIBF8C72OBJ$Fk1}Jy>HMe)`j6Tls<8BVo+flu`uR@x13f9t`4gVy4hlH z{pkOec#;e`FQ}b}*Oyp&I#DNx%e4M`b+&Aa+5^t=UbVNdf$>d=T{?|UNtg{w`$cEH zROo%J-;J$5@j#p=Xz=U;@V z<{*3@T)vBn*or4j%*-;gvZOap?5TlrhDrayEOw&|JTysv18<$3Iw2=bhTw*cBFnzO zW&cxVs`Z+6^RDw~#!HIOL3?M|6d46c+v;_vu-~5$pk(9X&n*^72!hoRFL!FvKDR8v zLeA1qvW8OQVB5o#wa;%?P;-(Bn3>6g1Ki}-*Mv}njnI-e+7wW0s8jRx=;sDUW;xK{ zqjZp=jhK1sJQ3X(a1tgIn(JvK+2hLyo6dLDcrAy zQaf8Z${3>e+9;5Qlkq2odRxMTGv{M%x?4?5cK0o7T#b|E>81(``!3xH5#;hgenAJ% zFmg%$jWl_GUvHRw9kpSz+`PR}B1muVoHf2qjq}ew(5!qqr^6bjgZpb-KvWda<*wFZ z_&{(Y8lQGErASA~Jld(P6H!bukP3PQ(o_(O5Ht^{S%53PuUhfcY^zdwLu=+X|BpLo% zA1eAH;d#MJ~4B%GzRDt*SD$WvLH}qCXBQ^}SYQhB+&|MNsHWA0_*=v4A0tAwnf~7FY$cti9_w-OjAn($ zrKLw1ouIf2{K)=Fz>wvC$=7HrT7$>?!4Xi$z<{w|7_(_{U?*KXYFs~#z6kM@4qo70 zu6eKNJA4%0IKaXF-ISgVL<0N|OIHugrzWT$^c3?vEIx$&MgT^%p`-d`OOao@Rw@O9@-21M$^HCpUWk}^z) zRH!o<2W~Er*$VLyt_Lp9Y(|?#b*f!Nl`-UO%b*31roNfAyT%CH=%=r8^}G7B5a|eu z7l6l-Hq^-H<^)R$Owu#DhqvE@K~0OBO9HyOockL+w;Tx+dK93)5SZa#Hu5cR_!6sR zxP6Gi5WXSQeapVUZS=v&$S5!3P)ZeMFuEfZC;?^OPgHjjeifvmf*#jZQ{xd35D*qF zeJGP&B+A_1*NFbyrA06ji71RY>z#%pAf7QqXV0W7D;Yl>`&gLX^sDMS2d#c-w>~jL zU1PFh~*8Fqp)zF%Mt^DG^Gw%>>XmX7Ju}csf?#jq~~BN zse!D!evZ=RL3p>L#)%lws;M+y$~`MJyNOlHU!hk__tdNYmPWA+2(?2oDS^q`Rn||h z$e!$nZ$P&t-gJN_!pZp!Xa?}UV|`v+FU zX#&o@kZ2%KMVtooW3a_Y9E$(>b7>tJ7ffh+RwuY%m3)v2y4O|rq)Y=NGpHKv#_HSFwc z=s#RwX{XQ=$m-yTtcX>AM;{hZ!1YM*CGQ+MRu&p)A^=paKM}1($HO3GWu6YsS=r;k1w%HER><$5y+d>)bQ^$`Mg5t8vojT<;LWLlhNUS zn+z!-#*m~e3TP7~VOj_2O&+&ro8Sng>1~LcP3rJQM00(TCnR^t^7|>MUQ#Hh6aIk) z=XL_nV}=F?yV14bzMRk0+8ca<*qK{BmVlS$sfW!6y8xo`GEnR*Ai~Fpm{9daIT+lt zj!vzUEeZ6!~R ziY2cW2%!GXNGmmwVZdZ;m_|fkIeMS`NB;KrfWBCQ!{%a@JQ~cN+rwq#RO!c#liTHM zyB(o?Iza^I=D!{uG#~p0$3q}h379wZ?xKMPttTaY@f(t+ee@*X%8Ur9JQX2IKkgk@ zEZ+(JA>6F^s>b*5Ec;l?kLe8Rna;v1_7yx{4>8{*JbxQqZsA-P06#IXnZc!3VrFJe zA^-`ibgy3*8j@r#f;;#2*3*v%i3lqC7<^j?8cRhbB{we^#Dj&}$lWEo`cx;s$U1a! zkT}yNC9bBXx>?p9?gi}NJ)m7cEcpow&Rd|y6|gG?G?0MU?B=P%PW=FkDMq}eQ29s%j9!IGlY3J8Bla!1`b$FEYdj{Fu1xca4ESE`cj zJPa~b94xmjUo=?&1oX}^9WqXJq)$|6!R7JDX0gaj@zAA zQs!*`FG{?r^Q5wV4H6pWE0$Egw(D@bv!nL!Pn%MIMbWJg#Y^x5!bAl9Rsh%eA!H7o zb{mXarlz_1`CA(s5DH4nuNU3#Dzy|})6vP)`HEYJSN9l02hh{fUfz!2P%T`*;NF{$ ztB{brzOOmu5>jy6>F>mUsX0LO6D6HHI4o)%pab?F8 z;F)`TatQeWr0N$JE&~%RP`2$zgQ7{v^82}YaS9jtP&*wT_AF9@(u*0#M;zkAs}mEl z&sGTzxrHuX-Xd-7yHK%*w~7-&TA;D3_+0s>JjCLsC9uZ*a(`Gqx7tkr&h)3uKU2h} zEWLc9&bSwz0u%SbPm^zxN|J6^NFTcq3m+{l3Trb#^F*9vHyqw`%lyD zd(WK|$`>}oC`T?4kuN~f{&7n}HxC#G5=5E-!+el#3Ca@-i!Csd2zH+Bm#?8=Vp14I zf`+JDm&{p$)e=sT6l~`D`Z{>t9FU7mZLwrgN9mOh#swC-QW^8RoLVBZc<`o0kEKM4Hexc z)L$J7DgDYeayA>-s&sdkkehLH|1d@^T3ffxVYrawQdv^M1imP6(W9rgFg6ZgTLA9{ z7$J~jQBYAgUH%A=5KjbLBw1PWn0UkwzB`NfmkWgs=?xCIJ1iNv_`*38n;M!=OJ{c& z7Gqp6BC(bex*L1u3V{cfJEYr_ZBP4ctzgaQ6UzQ_K^Bf= zMcQb*iASa<0iw?_H*=J2VJ-;;Ni_0DY0Sm|zaIb5pWd3Dwri))3GXox0`h2htP>Hu z5aJLocgBT}N3IVSLz~+S#(?9yZj6kEmev4*ctn&APkc5%^G1sz`lu4l_EfTigN@vWLuyqrSCHg;Sf>Df zJNUV}ub@(;H?3df<4+6ELPEBfqT$8?woATi#P7s_o=`scBfoSaYWHSk`*X0hhjt*2 z3@OA6%H1#)!%qPaP(fUhcb7CFK0af`4rqP1NQdDBF@a-!bhLkE3GKz45eTx^1Sryj(}K|#MQw>Z|{aZVfb1hp!`;} z-dX7!;!xCjj4%JgxW<5nObA{H#q*FTZ&LYQaS0a}bYiX{oTL$-irzqX*hGU?F(1PF z22%rDl1Z<4HfJT-#Xf?1J$+QO=TWrD=Mf}t6-*_^#i{D+gBB|$ytlU(v`M+SxdZrI zwJPdJZ5M-BHM94UoBE(nlStn1Li|6Xef8NU*bdYQOWy(A<|8M$%xmd=0@b( zNa<|X+3z64#P|8hL-xC-{c7)8{qlcAQ}0hDr%$gw`Rd1WU}YwkP3)|!6;5U|u7AYt?C9ju2;=S0iOl9$D;p0UB-_^lu)Y{c_-!PAY_^Z5U~7bS)0k%iNl~ znEg&O@SCznHUc`Tr;o|F?NrR%3;Wo)c_}!a4r{I1!K%2z{ zsv8jixP0Q0$|9H3Zq6aSw%bhrH!_?7e7MiDW34+r#U@bs{l zp=e8T*BIb>ej%f;JcY!!>IF{wt!(DxJiO(Y9}+`N;PtezIRFCB{2xp}?T|WRZgP$G z^XKQsE#+a?|RiDj1k{K$g(~s=>A;%=8}Q_AeEQ(r%|<~r24CN&l)&Zit_TwX=y!lZq0g15w;+Lo!-;&q@YSw zLAl`5zloi8e^@@dr7%=5Lc4m8!}n~44`C(7Dtr7)r(h8?c=_ZpZ&+CY`}}&*zU|OD zbN;OJqidOW`^y3k3feZb#3e-E0h=$-E?NWcRg&z}-!!!KcU|f`3otNHgAG!u`Df;C zc$O_pW8NyYDEY8S80m&&dC;56`Q^zGOREl`|MG=E%MeaWbxp&-Ku{YkiplPN39h*xtbW$6GmJsh%}`5con&CkHzyrjHb)cdA#z}znxtmS(%fSLfx8WaO? z8)RfGFDwARYg;r|#GCYQQ3H!GnI^114NWFOiF=e#=}moW+J-va|{*vV)_52x)dZNXVD_LzlEY^A(_qt5nPP)u)vN;fnwf< zw`b~GkeLFkivcC=-PL1MjEtt9g2GUHyLe4>FFxQ#Da&GOplq#o4IrI*S|mR7P*2=h zdwc%|a=6b&iz<2Uh5wO&;>&rbxJZ{1t#=2a?b%t=H?5$i=DIGJEn-oxTze;R8MzNE zM^;1HVZ_^|NTtKKhy;H+No%EX_;=pgUOJ6qAs3=9W?x6``<7`6o0yoiX2L?utzAG= z_Pcix92hR(^6&4L12W4VV|fLIb?LOUwE6jYrvMxCtibO{L=B6ZlXv%z-qbNr6taH3 z!6))U(HGLo_!&i_d#1gNN4avZQQp)qg4guukN_oxhwp9dirF=RZ7Z0ZzYp-pC9FJ& z+?|NFyT~>rI-aW7D`fAZ+eF_U?4xZ+vt3ENEmU)WaTr;rHa>U?7=j+Z&My>H>Ct|h zY59dBz3^Aq{S>g2_UWfFxr+}EM@}3H7NTJgSZ2pVT|!ImeVCuu`s(84C0KyicZXAI zg1V#(yR5RkJs@TKAayI#aippXwQyV%bCew`Bg-QX{Tio72SESK=`@SZ%cI-U-+VN| z+o^qD^@X}8vcM*K3#r!QBrx~%plwo447#~q!iO-%>5ne@7H@AedC>&yzF)C{Qm*B6 zOmc_qq5E(Tx|rR+gymWw{u`mT@rY zobjS3sG4?pe$#xWiK~@%>pJ#1_@aYbuE{s{SvAmlPXTd0c0Ty-yIX+yFmxpBNTag1 zeT3D+pRZ`}KhQD}xk-xzxnnWnE3t6FngcKa0m=nd>|i4VjuJRhz$Q*wS{hiV(a_KU zlUQM?;awLJ6Zfz>2658hqYrlFykeAUdLEf;RV0Bj_cSI=WG8Qr@kA6a=qDU6ufH-e zP6)HfNxgI@U|{^CdyuN);k8Ef+1iDvP7mN&(%t!-TiS-KM2S-_%9ox6H2Y6$d+?H( zC&-wn7h4sBBOw_-{mnY0TPghx>TOJKMNK%ip%4Wz6Fyj4dq%F=d7P|tfQ}*nrsXrc zu<+8qf8{o4;N=9{bK$ha#J+!%uXcf~3gjrr>om+jYv-JlZ$ObpBOoBW3_|eZKuqli z?L8j2DgNC~%&0`*b%xyQh!S9rW*0SD^Ot);JXYxi|JQGtEbN0el*NeY@wZfBLINvu z%+Hk-Ox`ACD9%ItwE3z9hg`1mPr0 z{9C3Txu@Br=Br)EEYiku+Q(Ap1%?JbISV2yO>Sx6+HQ~q=)10@!dOsVw9td3sla#u zdsH&e(Bx!gL9?f&q(mU=`+pI8EqXBv12LhK@cuap+5ex-QzW7I`X^a;yhlkF@(D)i zIn`{3Fe%ak;2{L$yta!+6n320rsOXdX(w*xEmw2>VuWQKZb|Oa$1hSdB8}W5TYH!h zk}PD>jAt?eerKOQawFp3>i-i*@y$a14fGiZK(l;E3xaIXO+G|TOsE>Vyni1(W2L99 z4J>lEkZWPuo0iw1$ALzc)a)H(q6w---qv;^|Lp1Ee9YureU44MO+yYUS^=#8Y7gn&8ahBI%L(pEz+=--jRb%+1%Am7)PJ*O4~nW0;VGSe@`Mgq^YOP!Lg*iLwbU8m|+}@8AkJS9i$bfGh&!7)(;2sRK0G z79i+JLFV783jBO^z{tj?ajwToh{@3{_3j-CLvc=yiBe~{t(~76f6b2%H-a8?fE5ooOBNNsi=0G6jFtS!0zUW!g#d=&=!3j6l#>h0V>GLr;`D3j ze27p{NeSvW@uQ~gEB>3>77^Z7WdIUcnR@PY#?+L-4R*K$tYXyeLfx&;vVRjm0;P-t zOODXPi}#~)9n2qFg-JW_Cz0sK05Qydk`uj5?+JAr`6~(l^rd1WkCLRTRI!8<}QZ0 z#+F(>^R12W35l;?1#GXklyTR|f-)8#jQFg>`{?eM$>o% z(f+VR>BtgJt>B(i{uGqJAMxAaLA51gTsZTj-Wf7dG3OvkN<_r(w9yY{(c5icV`qnT6YvBcBX$(8wm@4`T3QN%MM}{Y5H#7s($W)4V zUCCol&7fCkU@}%RGs`2<7zrjVnG%j6ev@nsLM!vDM327yA#j&7(StH56}H?VG**(A zN3IA*AdQq}DI`IE;dOo3yzxgyP(P3>qnb4_FrXw1)?=ql?3sYr*mN6hbZzcudf%L^ zzR+*qOZ)duZ1WpOhd}tx&(Age-_u)FR1JABWQtf}z;a_SKDB~bx~3W&u0Ulv;XXFo zJjfRSI+QUhd3CamdbG##st_h*Ty&}Vd=Oh%`-CmriBp+W=E!hvLzg9B!2rkSyE~pm zJnTpQCC8IW?ZSO?K+Bj3p_1?D$btb#?y)7if8i z4oPzB1%ZszuuVp0V{AFAGmTzAfLP=Tpy^ne#>Uw*R%RgxnmOMwjbVRQ#S$mHVbiZ0 zHDXOv!rFFjH2o1p!g~(lRSf|iV0F8>uhS) z_cM8&WXax{{s+4JyxS4uO8P-Z&_3GC1k9a!i9@Sirz7oyYA1K%w!fW(UCK2N@MWDq|G)j>K=#CqTA5emb8Yax(x`c103OavzxiK|5wV=1-?7 zM2&Zf?(N_uJs0o8_4XW++6PNaTSsWqG0pziwRLr4^-bA{?+X3aP5jWUHmb0lf?db9 zfm|v>Gv?&{ch^`PZDuqi2q~~>@dd#C^`{{B*oyJ+&(xDZx<7Dys6PmRdm{*yWnp3I zjSFU8+M3P@E}P)tF;(wNBX@WH2oc7hW3d?Qg>kVho1NQ+hX@A%%at3L3)#zQ6XA4m zO2twEef65IAR*-@*};p&sVP{bbv_eAPWJL&2if(QCp~1mXk6>XwyJpR=79R*{=`A) z@G~*707^T))N|5djsD$7GT@<IWOWmTpTWK(;cr{=5FSfK$#GCSwehEbZ~R_^uC zW;8>dqjh{R8R)u{LE}_wF+p}>-3_Bz*+Tr_zvp-SvvZuMuvsfG%71S7w1M&G7*`@x zCBja|2pa7z>d_o1!L8@>B!^c>^~by!OM?%UaZ{!y&?J8hYroCRn@^n_=v2u|=K-P_ z$ee{J?8;rQyNBVD{_SKVpXQO4#eNijxF8IctT6%M38AEsjNDQtCNx{?=I~8w$?965 zDnz^@W^G)we{Gnv91!;^`D$8-kDbzoBvjqmBuQNo8nW~zI>r{Xm3wrSaF7~;sou={ zq%D`qKOj-XatH8h>ztT*d0^9^KUsBup9{{`zhmY8SSO<<(Z;+REw`5K#p*ZzpqE zL=B({%%dz)09R$l`*R^QN2!0m?7781mul50sm0HN-7ZO|CQQvT16hMU|A_!>&6d0& z_Xa&u`$6fJLw`maF1$Jh-cdAq{UDI8M10THUjq86(8x9HKL4Pl?4;Tj-gwiW#X zyJs;ju8#%=W1^&)wxzKP6VR2~WqbWh$9pSKwt~5>z8wfinB!B#9XYKpjKh5=Mwpn6 zeDNSIg1>{P5Cr)W_u2LxE`sOOirsMdfAE*a&R@BU8xDvWVcsDy1t!gpj@&M`Pl?~Q zmy_tnVi>+#nE}Y?|GOO(bp(4O@{xFK!R91a$V=APK^;0x7z(Y^-|^*vc8HvWB!W=< ze2CbmCC3wC=nfWgAg5Tmo6SPM#nLvxl}!F+Ii$+h*%ZoF;-4$*&a139Ky3*ui>f&J z`RVBkz)DFPgw@ zzB2b#%H3hr%|!RX@1Y?-)WRcV_Q>=o^%S*#ANxXuMT}_iBhB88F7~1V{#)r_6D`DV zmt)m{yj)9*JR}HpvGR(-@Pd(bmXphJWQR+xuBHa0(ssjVI#Oj?++4Clb1RwroZR_) z%Y|BI=}u^?!BKMwJUj~o$YQ`kmrfY>*?$iz`1ba8A9K86T^eEhz};xUvH9zB^i9!`LR zD6Kg7q6)_Y7(+r!9|mAXu7VL zD@i~Tc(S}#R=`HyOkj){(LIU8h>0X79K4q~c-BH2c>+2taF( zkeTV}+`K%-W}ZI7Zy#y?LD}M@=T6A3w&!poc#g&(dZ6eQ=f8#Fx4eZ{@;jeBoc;C< zTzlbb8v4Bl6vCxn`(^;Q9e|J|k5L_IRy+E-&?Ip7PL*2|(b!Nm_UaOwX=}}4&fN$w zkP}uPlJjb)9|8f08bOP{CK5U(c%x1)Z<3r(er(S$f44WEjk&#q*$6c@U^qf`*T)daW+rywX!LJS4 zAbgAfkzAmkZ>ab;H`f=(dSd~;Ka0GQ5<&;8+h03Ajc7L+sPP8R{M=ly`J|8u5PF@@ zfp|{rP&IVG<}ZElC5tSwQJx_=axImGcl-WudKxb#T~Igb2ISdzdspJ+8L^pbEYmrR zy~O_3#%HanSX=K8nrU{%P|Be2QP5Igg-dqp^QDpiUjd9AZJ-;&vGwC;Mf6*7Vril3 zL`>WPGmx$#)4K*Cc{z?x#GcZIkV>`&GS@5=6J3Et#e%ohg9{c=dDvq)0btt(Qk5W_h@DBWGG$ z*MbCN^`vCtRDXA1bA0z_N@$n4WPOVeVJPR>_kDg}W9D|Tbp!j&Y;1GVwu9((aQ$Eq;S z&bPd&w$-4r6X#Gat*gGwiDG&TC;DH-p?PAkyjR#af=%3&G18#zh#06QzhIfuiS;_R zBkoA89{)=9NP!hwtiJdWbOXodTFqw#3@d7MA|qQH^`j>~$jZn#EN=dO11`Cw^t7~1 z6$)=Adn+p#i=hD=XTPI^+zrN^)PsdQe`<%Q9Lg~PcJ^^FBz*RhS;i#quY2v~c{a-G zl{Q#@n>5u%KkW~lO`cO##j&J#;MQ!vg0!U1PRVbL0vGfdLwtlkpZL*jw5oVZ(X)FB zO6B9K<~$_rlI3d&g8Es=jru;Rf;tnE!$v~V+K7Ka6(K+?^4SOeuno&fNA*Mes)#j- zYocvu)RWr(5~V`LSbWi!$Wlv+&C{e;5#BQCivc;Z*lxE-NyC)+IYDPtEca^n6d#t` z3~6{K2qqV`!C(rZF>@(_;s!0uluI+h0Pgy_x;Dx`bP^NRSNAYd?PRaa$Im|pOHm2S zsS^M0OHse|xWVUz9)#ULCFkZv3T*-&W;`*RrRC+ovu~^a8K5BtmZ3V{0OMF#q|CB*S!s9irvY_x846Xc&NQ^hQ{g=3A@7e}()($ExaITtWotrnH@y*FK!FnBlimH4 zeeF;2pSN}-hqwW=!RLqjy;0Nm(dHiZYyc2qD45zM{b%>ts^Q&O;j5R!tMC8bu`IwP z+3fUtj_mrqQ(1cGri8!Jq`hWWYcVw0fgiN4oxD0aC9s zG&a}N#2qQEpbY*rR#^vBXb(9bW@tgYvMV|(yIHi`aA^Pl);2UW0Eb?0`)9qL@7Fow z=~4|ejs(CbEe>J=H@u1D?ziha@4~YNLs3L!2yeisyodDB ztyx`@;kiw&Mya^GycMh&$q*})EXKds{CyMwmnt=-o-JL3TK;mn?q6R?TEd@4vA<7- zYP-7`f}9&F?=QuX1w0(0$?Rf7DFWuIei^)z$sjG$9?F4@a^2{U12qP4V5LXMNJ*U= zb^R>OZ0u}pc@S)SlutLqSj3;757T@VKUq8-zh~v6T|SH6u!`dN_ud3z+*TD%_wd%~ zJ$F$f&qPwM2kBy-&)Xo3fG~VKR08es53`hEg@u`gpRx?I@ZeGKi>PsQ3#AGFXaxu3 z=rV8K_m~Y3@Gf1i2FCVrb+NafSbMZ_Qdx|~ZvPY*_DrAi+;SGhOz`r6IF1z+X|u+3 z+$UNxj^Eg=>nnvc^d(~TMwuhyx`rhB3b6Rk`Sm1?1bq!(d$>IZ*NreH9z-Ia z@_BEH`t54InW(mgOY#1={j8Nkhkr-t^Q(Ey0#2Lh-4>rB^1)-rS!vuK=~sj~7yT4Q6&CDwBOE8HO_{~!Ao#=f4}_sxW%!)_ekCRsn(FejdBD>|At}^ zw)XLX1dFo=iiTo)Pabka81=p*5apoWIi=-4zL%q zHK$!o-jB=s)0y6&qG>Sesyj)nfB2N5L(apUZ>Z5iKw&FjPW2`cyF%WEiFODnm@Qeg zqOLI)v$`OmG$(*#6bRY?#rU)NaxtUO67tHAh)CS60zqGsCBZfc_`}KCwf#y&{|mJq+*@+$Icw`- z0e%&4#lqzuQj!xJ11mWOpRI8B2$gEsJebCw|7dk8u#6ifWj}dPyl!jI>$Cm6>CIT+ zy%GElDQ7;t6k)tHjD%r0{!GqgM1bOBra^%1XXTtx^ zIyb9xe{WAhLIQm9f-SHw2YKZ$W)?QQb08ea(^CuMC@neJQh=GD${>@8sdkvOM8ld+ zPbCw5^YE)n39$KCC%@UMfz|dORz%FLr5kk>;Er#JqH6r*e|65KcCTa8JKV5RCGeC@5!dWFTz~#V)EVt06R{(H zUjvqGH#awWsAeZjl`o7?S0Q2B&0$nY?O$OooD0FE**!jHV`e5BZ2_4%6Cq~i z=Giea?+;g3p}!~ep@2d|f{!1JFvdR2EL32N%Ke)dMo*phLc3Xgzn1*@h6Whttbd_c z#)fI}s)g+hrW&Om$HvCq9TRZb83)o;T8(@kB$1_TldY|hRQtb8!MBCZ4qR;R$E9i^ z>wwfbJT_Kei}1$xrd-riBAd-H;tmPXGh9?#G=e^@84~-NRYHPChPk7-ZcxSeZvE{l zq^vZ7?l2#z2IY%-XsAF^LurZarEE+r;}_*K391%?!D-mNSqRDC{@QgTb@~))& z@yZpKsyYu3qXW%0G83eF=u%fS63sSw{M~0 zz4$+Ts9A0P;vJ$-L7}}@Us!m01Ok$;R=Yw!vW_4B&Y`WQ>3%hCMsTfGD9CO=d%L^8 zAIOx$$x)gB;>u?#ImdF)-_0?8Tq4%&m$|a3ttozuxJZFL;-E>PU8` z_&WJ5NHNcHDx7^>cPxk&;4IyBfI-h3(tIuN?@$qAsaCRcy!yr4`va|WG>^(zo$aD} zh5k!gJYa%bu=TC1Z;A2q6Dg>27IS`?35s{zXjn7==4b<^AU&!htFFF{yTM#l72CYH z0QID?GXFp!qfAbeF6K2C7p^N!afkdq=kd-C5(>(%S_Bfh8p27aETeCTC{*3PC1iW8 zkDp6sXT1-LSiT(*mE1AK;;iLqlSv{s>eKvq^-0!}fHiooPY;j&9REUA0!(TgOUcYUcP;dp8t+@DPp<&>d9_I2 zO+qUdlXflu$&Er^>nY`)a`&5Y0}sOBrtk%^0^ndwMoDHNC6k3HtM^gfx#O z-bEk~Gt>~?CisiIG zQo=}(5s6Lgtn*-km6Lae+7e$=vjqDGO)4J;hgaTYyRS5QqpVcypPl|pezf!8cgiNd zoumJBkNK^Z4mQNfqw*LOGdhASX_ZH|Smzo_4fIE5i27^iYUq1~@oT}ieO2;m-3mJB zXiu7N@@5KJA^qBsHItuAEEve#kr)c->=?>IwS$aR`3bVP4@UQrI+DUe@R^dhYRW3h z$g5&NYfA^EdmeJHfgDcpQ8V7vp^rH)%k=Smqkb*btgk)!+dfqcyKx~|x%`Ro@k-xk zpROrONb=IsQf}nte6__{4m}BrySsZ$7{q=Yl10T#(uO}Oo{}>D1Eb~Xa2f5>!;^KjznY7LYKe}nX^Kri;vy~|W>5G|C%P_qrdnO3&tmt*FH^I+C%av`=0 z%Rs%buKRwhNYrprq1)JLzk(+LE{`Q~TXa=*(7J0T!G_v)={3NGuA0@FE`2C@IOlEu zD_}R6wa{?;R8VH{qKgd-2pIh-Fa>5Vd&_ZcDQW4f$3HcexZK)JG2{Pi{{1HAEYnUI zcwlr+Sy|YGrwv0a2xw&sTvT(& zrY+xKKY`H|AWP_Z@_7YtUXOTRE&=+4LB`*1{|Z!pV%G6D^Gof9KYH?l>$o^#<3)%vu|8o>B3$Pukw?wq;ceVd|m7Wd)?@17#DoRVEA|n;vzXzrh(zsy=XysA7@~*QW!C&9@)GXLdG7>R8G_laYwerj%kI{eDJx`eqjJrNQJ`V^VGLe%9YU0s@!`>>Q z&RNHbUeqWbQ24UWWS2jSwtf{NM(vFWi}CN6XhWxCsS3e4L())Erfdohi-`F4M9M7z zdA`uef|fK97>En_4#9zaemaINFNb9l%dZ$wR+=@=nW8tS#QOQ@N=($r>3!#e?kF59>v!gJFR9MUi*Kgf?l>NdU`C- zuTyy~9bL{8v!b!AN?<@A&ln2LC|j1tdkq2~sZKvKjl%}+#sbFV4Q;MCE@zj2NT=+j z-UP0!?{VbldiiPe9Bp9T66bu(4lw3NMnMsL$vPO>+28*do-y%?(LdpB8*L`mx!Nn5 zEkjX~T+r}t@mo%=RI-;LF6==Wjqgl;V0lJ>BN06;q#}1z2g|Cy2FJT+5}Y0Tr!js^ zkm&B8M5GQ~jWK}5rp4%BxelYjH{69><02**vsosbS8qN-B8f8$tT3{&PfkuMml%(_ zzy^Zk(h0}4l-Zy7Lo3xh*&FIhs2Vl@5Dq{rPgSJz4g$1K9x-Zan7rMLsdx%^{B@36 zn6#uM4-kp!gUL7^^mSrts{9d7Y}C|K#R7?&yGeAWR~upijyDcSAP{guB*_KnNr7_4 zyJ2Gpgkhl?bYweeB(a2Cw2MqNIzB9}qA4i|OG|XFu{Q=$dYKiE1|#WX53&CA_`kpP zL;TkzWWY4Qw!6JS&j_h0tTcMv`Qry3Lxrhd$kQIzti1C1FSDUarTJG$?ScsurGA~A zE}IAxm)~B)#${R&Pa`WhCdSZoyp@Q&q?%KQCn;R*zDzCyib54U6ui)0qtBl|?@#@E zUnaheru*9?&OjQ4o61zNMu;KLLAG`_Fh(P^jEW;&*mH7f$|T8if3|FRSn&ab*dz*^ zt;9Tqe&RO7p$&I)DNfhPfJ3M@c~4<$YYUVm6zjb_j<6Qb+wFJ!+`qMi z|NU~BI!66g1}t5z-~rc;3aAMUda!qnXFmLWFRn`qx{+-Al!J&JELQ-<@9pfUIFI|W z=>OK_%*{rZoSK}T4l=g{t?#5@_Msy<_ls^rs_S8jVqTx=UETSgffm7J>81i+8}lM? z9Yf$-x0Jqcb!7fqv^P-#I7>@FPI|#vN9p4M5@0PYi(E8ve0IopB|2LxNz{AGnpaQJ zXuE$+R~r20$<3Scr7j61RxUpGRtXQ(sO1{fs;aQgs z^B##`$Xv=LcF_Ms>@Fkh)Fnq@jXeTWQcs` zJrSNSo@u2l@EXowq6*pk_lB1>rke-xR>tZJfD`X*74Gk~+X`WD^8GV7`<;frO# zB{n&1V4MA3-aPn8O*ujLs5l;*-u7^&U}$Ivklev%SGVy$e{SxVpTRPRFvANi)%|f& z^es$Ty9PB-;{b)Q3q*oU2{_o3Eeyh#MQ<(;yonc=JRF#*cmAMi z&~&-2HF6KKWRAx_2%$Y$*YeHU&UqdSNA#$_{LKBI;{~zHF8u|*TX&{aF=orHUqC5& zc1`eCegskyd&qBu9SWFaoG=#Oj%+Z+v7)_e!djoFTu1hn_wN zso-b5SLm;3$iMh7S71dF!GnnKQ8*mVt`OX69&aTH zs1il&tDX6-d~jR3`&@#R+IAh4eDJY^h}x^58?#-NUOU^?8}3{;DgO{^rrbUa9B)-G)C)luqwvZI> zqNn`%a^`7#2d1r7N&nS-gy~2}uOgH1_I`ou>37CoWO5)8!ocDsA!q9O72rzIN}^JA z`vWe&!LrfmH*5V_TC^Ei#zI0XCyy+oY?8!J*(9$+R`sJ)gH?aM&CKhY!_vyas;I3T zOI(wg9;!?cp&~;G-)yIwEGS3dnW_jMi@Z`ef_p94Q!H-5QtM>H5c*#kY7?0fd;?yg zJNhRciFeUQM@LK=<%Pc|K-YA~(Lxp(8#@N5p)%7AFXrjM$r~_Y6~{S5I-bn{i68t3 zg+)cHGcx^Q;QCNSN3q6(gk?(nQhX@;@KN#dgX;}jt#)iFoP37s1od@p;8lVp1im!f ziqg30H8HUzOaplzbcDJan|G5(b}Ptgkx^WtFI-rg=xWKVEprjrJkalk#w zLOCFYI70M`?59t7lp?RYc?4@#f&m$K83 z?IBf4#U*ttZm;9;Lx>=rN~$gfKlP)u(jU68-#s_k&|=~l2m5blkt6sYl}@?6Ii3GQ z{!WIf!MXpE)BHA599N<@<_k8td9~;B6YwzF{KzmKODUFai!5j^sJQ$#Ohnl8ta}jh zE`b^qIUy5%wQ6w1Y75jf$T{D6LdWKeb#~vR%OIr4{>8{Vcqa%?61jp&5QxTYX*_KsXu+9{2)O zcTYB;6X@p1+$)Ni+}pelwUKoxT>W(+AV~K3#=qo_@H;23h}n@I%s3 zgzq%48{E%~nM?Hnc`Bud7q#Z(RF_oXidHk z1Vz|2x#V7415>D^ku4y21$bO&AU7Xh$qyiH0e>SWM-72xvZ&Ey0^@}%K>o=29Sni1 zMxjxGdS){906KBPl?0{{wa3vjN4BA2mC{a;S-iKE#rvx;4D`nI1}_guYf7#F5WV2xC$a|6U}g*@B_+Y+20Yw=C_E#B zvX~8+&9Ugz8N&Edr2(Y{Fz2{FUe>@jcX#J|aOL+8@qKBYB)nJJp?9_G&Zz@X3UYIA z?(ga8>FeuBzduTPv*|YY<(-&A>K}XFfFCf1T!;V*3o1H2Tv`03%zY04r2#fj^$WP0 z-=4Dzz>MHcC=CPCz!C0AQyGu*7Y#I}=)wN}{b?WNi_*L|nzRrWy)C4}_=PwE%mbpL zqLPw`O34TbL+ioa2@LLOgcpBp06%WvmB_@)>o$z}VVwvVtHG%PKN`G|;5c94nVAIJ zB6GX`TMig$fk=9{<9{OS0fH}m)xes;7j3r~)bm=GM_wAiaVG!{WeZe^Vc6UJ_u0mP zw2c_^eLf4orohtxdmV=*K*XxAsfpUNNJ!= z+l=&>m03v?GT!6r_q^{P@8|P-{(C-m?)$pF;~d9%oX2UDR8{2-IZ&EC4?mRJY1!Ba z;?Agr0bAeq_8*jq-W)o#Op2tcBcn0(@lH93I|sQXHpKZp3Y7 zABCHm>GG+F`7s{hLy!7G%f;afXylCuax`O3zjf!1GMm*RasW)a`@6ravGmLa+mB=M zm$$Rgll0ys@72eKu-ZP|=PXb?a@bZwD=;w7#wIIvhXSK)>36Ho7d4wUd>yfQMhqm! z97Q%rU0Rg!AG!n>8tcLL&K*}XKVMk<Sgh6=6}b7cc#G>n6{i>tDCVt^%x zg@M6c`@vx2F%a~d%pUiJw6E%ipzoE z>E69`dN`O&7i&E}wSXhve)iGfA<|_}G$%PbPrVWdj5)N9ETOxn=fz8#w{fvLm9l5@ zS7K_d(az$w`RWf{#JH@77EcbToVm-@VaO>xJMf6@Nui%FiVhz-JMl9dPNQbNetsjh ztV4J$h&`;K_&BE9y@750^DWoA+1bjA!@+3OW#BJhpquHVjEjsQv^BJv zMuY0?-dEW3Z|_ttWf8&Yq}K1kpo~wuG`&H^o7R4;w~z|Fzo^{IvPcQ`E(3Y?W*pW zFx%b|^yY~Ohm-LTx{1Y1AG*1_gHZeC?vZGJXu0Zs3F(2Rzz~Kn+l;>F<%m1e=c9-~ z+OQlQtHvx4jZ-(=??0DD3rfcHLhPM(UiBfwjTCk zb^1+AG~v(Bds$%ez)cb~@Pg0Q%drOspSMY81Yz8p5xF_U=**c|u^T@XJzuQ2VL-AVMv z0>3B{Y2gcaudpMKr@=O(H6C_Lbe>u*zOzJ_MB<^ldK`y6T;XoX<>=tV9O3=Yhw3U5 zWO?4&dVa9R$Bg@ybwHZI)A~y2N%e`!$#MC0+@W%bUdZ{yuNA#HC-tSb;lq)SWcz%# z`W?c|j(_nM4F1ix#q{|LJE7D9yXTC>b!$&ZYH5)SOX7XSYbfT~i<@X`(MhOl{w*J* zj#G`JMmVU3zo$-AgfqFO?Y?VwmzHN#-uvt6h%?YHGq-v$t+I0@dRf6hlfHC|lS`bM zoJ-$q6R+#=K3!we*0wfi5fbvuf)Qp~u2Uu^Ced8`l2C^}TuOiG0jUmM)bft3^uk{3 z>^m#^L})mUs5%b?{5qqX&3cN)=1ht83yhq(rXI@5OA0fr=G%KIyvH(N!Q|6Zc{x5g zu6?JSUcrGi3*nsg z>0yg98r!EQsf6+GahjZ-urM3lcrPtXnK0(FyN$oegtOIe-XBVY3o%D_lD-P?+)UZ# zqD3FZ)GJOdIrC?pv@MK?nQ47x zoXv7;lh>=y413XS(=5Ncr#?2P&w#%~DP+U+^V{yTMm%fB<{#I3BiWoG_}4pkUYnYf zg`5JkYlwZa7kB9|bEuB9q021|Ee%s%p1t_`bG%0s2%t9~0z(k;n*eyR&d>^w5EF^u}SrcPksx|TqI z*RQWyBNy_4-C}!{LtVA zwY>4QG@S#4d0w|)H7n~0`MXSeUH7&snR)Yw>V;Bc+udR8PAG@9%Af5?z5sGLK9T0F z?n=#djpZnhrsP~Q5&N=MPNXK)v5+EDoz6?p3F(@V6eGfQW z?REa+E33t*4aqjomUEc6#0^>^AV9N)oU~;R^^MT%=smoZ zJnAt)_-r+Vl}A{guG<^{?Ka;A=(EYtCGLloH>1(YoB?!O?_BNNl8XV z#)vOjntj~9nAS-W*R32|=eD9e4pOK)ObOv}xeu(at^#EPPQbpKl0@@frnOAz`Cq*c zwi-C)sFKl@I{#wZ%?;mXCMJZ@!uXTHHFfXOj)7vTd{=%~o79*8)>FuSHyGtdh;pvKxm>=pic3 z3EYgI8``hAbjWAMpLu$zqP+Y+JBQ>$-E*Hnk#~ej^e&KoCi!`c=->K{#UA|l#lrPj zg;U2)PIYnpxk*}{TVf6zk`MFqpY!>M8m^b`8Y8U^U$WpL=_cDBsXik^E$RR8?LN7S1n3hTlqElpXeLiNK#LsfJ4q?KBd zvkLndJ$FPb$8dEXjyYaJpBqt_;;eqO^aiJxsdZv;v1INZ=8VJC%#yE!`l9c>W#7EE z;8R56V_-3pr1b6L;ChC zX*@%dV!q`|mTX(f^^snWIze|H`MpysDlP6IVT0!p?S-RfBQ!j`>p&*d_EJViZIQu; zm=XPHYc!cHt*wuqsQn2JNA}+FN6{el*TyA`@KX+>;jcf;OIMXV-Z$(fWvDK5Kv;y2 zL#H&o-z97*lG>K1h@mG;##W?P+FnLaM<>|(Sn2G!fHI~C$;!r^+dd}l()##F#Z3Cd zo!8V|H+N;R?4+*2%lD>jWcD@y&n9slxc}x>SDm+0o{{B%;J_0HcKYD zTi9D%1rT>6fN$xEs@q>49`?sG*fTUUs9u-O4~-Sp)hX+Rl62%_J+8PBcWb7~kx|wV zf*fWWdDzB>F$E8r`ox_m9sDJ})PdsUOyFl8(}Q@U{XA?=mG?QE?v8`ZvfLKmkoG)t zefQ?+*&dGP$F-xAOcu2}?G;qh@A(h_?TPLbL(` z0&Ek5gO2C=?;O^8Zx`k_&ZeQ5Z&)WgdeX<_()#bq%6c|%I_~=h*)$~)G$RTHO~O15T-P21j2}j zrp7sA@HUa#BdUJjuF~wX17JI#>Zvy5Y$>vE4 zZj%=k6T=3y>oi4z6c&CMm5bD&pQkt;;;HZ~lHhjkW%7%hO2Y6$LFWyAq(a)cKZ z6>ax~KDjjh*5UlehSSe{7f>&$vCOy$iDU=~>d$Xw@P3b)ovCXe;@d37bRny`-rnA3 zADJx+Uw(h9Q~q@4or~Nzq^T8EpEk1`bR9&3{l`m`bocf~sc30v#5_&_#&p!lx7nC# zGO^qN-+mn;Xvxjpot-irLpjX6aw_S0c`C(brM4a{XVd9rzjBFtCW#^$=%-Lrur z-{*N7aAEhiLfcxbF;d~mN;UHhj~!Co=fo(i}GYQ+df!>m^g4KaBMP%*rE^{Q*_QrMG8`{6*IZQJ^w2`<)nY7&&ECl}9K zSPW{ zdieSMj%43kvIbM|Fg5I%wMZ|LJ0ruoJ}$bj`ee0kHb!AO&-om{Nt-?AT}?`tgHw`9 zvvD@WpMjg5qoQNDc&JdqpId@Jqy>9bS0x_H6IwKeZ$lAi&L@Yzsv4;pySgEo9&*?u+S6 zB|=1r>fo+j^_oip{Gp$3hHcd8fjU!3%+lcSvBaZ zb0NY{yG?5~_uH{oHm$OsTWjx`=?-5}St4=K($VQOBVd)Gmu2~D8aw&2b0LrKgDL8x zCx}Da_to)+s?^BjWMuF8ww#4uzo1fvy_k4qCb*x?%4!Hq^5i9OG)X-p))42g>#osJHe9j1zE?1ZxQ z9?IqD9`ao0xk4KA8E%1NESH~9eFLwINJ1kxAvXNk>}pt$V_3o|YQ+VZ+Y@^z=qv7V z47xmBSY=%#!~7`3YUTNVmkms z?VKkswZeJ*)#}FF9M|98!4uJ;Lv|t@ry0tVE(dIym#uPS<>z0W9Z~;}+me~c(dqgAq`!M>!OzFcJQr&p_t?_T)lrsb@zP>XezrdpGwsd$Or&fWicb3H z$^@A2gHo{DYu2HV1a4J5y*IFnxwi=WLp@gzpU^ZP#>`8u%;t3JzahAh$27K6FGJpD z!Fu=nDH)g7^?^h@b~P66ufW3~x4O(mJ;epTG_@hLX6f@SM2W{tOnSW7oj&a4Q}Vw_ zT;ptVak0MU$qqVoyR!iShxJ(m-)QOUzs1$X7&9SNvdtYNwL8g`wB{if$C_eCDSZtn zyLzU~#T_G~aFbJIb}3foU749mX z(VGhvWK|IPqNAfTgZR*^25>1qe6K1Yk%)i|M1HU|g3hWOHrU3+m710|iTVmqxcBc{ zlGP)AfbeB4lLp%+m1Q2*qT$TR;8(JbB1-iGrlI6LpbnRCn?5e5ZK9QpB!v0T{f~0+ z^YJ}2P0dkpXbfANx+9aacJ`U)badlR4yJ+Bdl#1#DPRZXBv$TCsCSXO13(I6d}AUb z?-v$kRtKK2v&*ilTZa(iX5yU3q?eV+9Azn_J1+iqlc3&Fz=h?L$m!V~&r*(a3mJs* zys%m=5!s?(Jz>3ul&eH3hW)O{*zHYpay>oAaN)4WNBW<8 zzPE3Q36fcelqsS26lZ^%WyZHXDwJ+IT&f}6%*sP;I{w7Es{Mq)Uz?I zetGfxSmfftOqkb}>te}23egatmXUy&<{;ukkr010H8jjGEPO9#9eoI-aHg+x3~*DI z+yNTHs-57`ykve0e2?G41Didw0@B={3v>wGfKbs#Jr*d;r~Vxca5j?)K6Ce)?VtEA zh#v4wKLM#xK0g1o6(23m2wx>d#hQEf>^o~+i9g4kV?aro^?pJ^0-_C!&?%TI&nYb2 zCo0-vs6K+m)qwAfTa%ZL6Aw!-*l<)>Obl`uqLPwz{!22tfR2;}1xMfmVTjl{ymgv% z{rnE`EknX=Oq_0g2wBVOyTH}01VBqKXzxX!kc8A~in(e!4io030Y1bO7WAVW77(~I zJUo2mzm_KZ%>&Sh<4@6cMm6qexdU_UbgH#`1% zU6*TVSccS`??Gy68S%(J^7|rec)^r8`{0!(kDS^-*fvORh-PO;{ z$|@CRsVl>eP??xg`K2xM+A?3|7ZCe1cb%Xm!e*b8)YK3*680FlbVp^~cy4&4G-&y({f5v&g(Tcu2K0e+u+Cj+0 zuYzULT8_Rv>Pl*-6jiKct^eaI^$_}~NMRlI2Y4>;Z_Of;X7Aw8uYK-aXJ-v8cw6Lsrf`0Ha>$hpR<78c_QXYiWG_TUg{C-cy;w6WCih}$B!L*S$BEa|5INE ztC~3d~{5hn7l7uyubsbI^xmj^mJiWo6e4#ks%5dGUlEdzwBv{@UuUM?$8fMDuGv0m!g(bY z#puG}X#_eqBdHfw`$JeAcM1#BA8hWXq_rZ~GH6Ch5;)hqS42bvynPB4K~gb=IO*Vl&`&v8N|^~k6wkeZbKwFPtF zOtEa*WH0kH?KS8Rx|Sja!(T=il2O4P^h*$B%6fC3%VGy$>4+ zaSi=1(r6E8eD#Vr^FIvD%(emH2he3cshjl&fQR(>AGpM9&}TeTB(R%@uEcl2c}k;DcK;q^$kR$~ASXdNjvj@oqdit1g6Jn8>w#ZM|N}+h{81@%MUcgSU+u%A@ zIH55Dw%T9-afRRI%PMZA=9-#e;zumx1K0eK@vSU;|DMQ3SD1ABc9DQPYgdvm4||^* zNW`#;D5l8h$nxA!F7k$oCsqv8WlW5X(M@80K;cop|LXEWNC?eJ&O1>MM2pTml~WBR zcxEJ$@7cqLw27BuE}(mopHDCMVPGKoT>gFOpbUPNya70U%L`7GUm1UmyVc zF+Z<}cAafBJX104fFP0EwTZ%7U=XV}-T{0|RF7*RXI$;KxpzdFUX-hE{~c9X8ey=G z^))_f+S=Npaw|)P#iB1gNk;gzqa7aVCW>rvQBhlhet!RMC?;uD%cmCO+go3sJ)_|C zrKYB)X$NIW#1)qQQ3g^-7XI1K6bc179nYg6Ko79ohbr5k6qoObS#)g`fGPy-TWEMR zG&Ee^mrueW;*VhPyG;cIW3S^{-m~znh{MvytCD-jLwJ^mAdo9b<$Ln?!TIo;@O(9K zyS;&$8d6@4%Uc1M(x=G;6%`kUlTbc>2>ST_LL33}{`&e|s3ar%2DIxeR9S#d>JG*! z+V}14vG|)(x^6kQ2^4y}Zke23L>r31k@fj5{jvC*rb$e5(bd(RnVIpr=F0Tv@IH?> z-1Weo$6y>zKQ}j5v?6p4n(CW3DY+A8Akw@^*T;yrD`66mO2yhjzXZGTqBO;?#T7yFUFKtn1DrlF&A zeP7Nn5Ou#bB>!>j4pz0-E}xOLYV7d@g`hdnc*e=MDHbXDsHIxYYpGl9Qj-|AhEn~^ zK$RqLF)d3YBkG*LU0h^0UyH;@vTdG_CqP~Q^5yE{;%`8XK*7Md%AD~7*cqa`2aP4Z zHx4LjXwwTO>uYJfA{Y+*{D)7!IYgIq{W>QboAf_?Po6ysfCYoPN2#I$+#zBsQ3;7! zOg+Qs*cH&M6 z;$c()y`$=c^QEHl{QKA|@blL2u!fW8(AbLWDS>_E<$a1=2nwY>*9^Xl>IW{ff7bs% z>EE?|5Oc1vZpZwH5QK;VlOP!Y-7%m+pLY9eU)b55nE$(8RWlgX%n z%?TX)*D>5j%PZl;Mn*=6i1XeX#CVVjMNM|Aq~*-y#UB=i*<>iTNT%?ti z6*RYM-Twir3D7U=4M;^`F%SFv@5^(7O`_~OKLLaqDxbiBq|YY94!p$$$DLth#5me4 z5&<}1>oU?{WdJ;3VD|^Ku?Yzg;U7u`fEL*m`!xMMA*g(B39NjFo;4VAh~n)_8;R^hx@^Dpkv{?>Wu5GoA}?5f8FqZ|A4Rzf4=rV z3oJ<7ls{kfUqASd{|;gO|369p-#+;N&+>oY{9ix#_gU;W&aG4VYUwObIZJu}h5wEo L(bFhWv%dB}cK|?= literal 0 HcmV?d00001 diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index c9fd1b300255..af64085ddd19 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -23,7 +23,7 @@ RouteConfigProviderPtr RouteConfigProviderUtil::create( const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - RouteConfigProviderManager& route_config_provider_manager) { + RouteConfigProviderManager& route_config_provider_manager, bool is_delta) { switch (config.route_specifier_case()) { case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kRouteConfig: @@ -31,7 +31,7 @@ RouteConfigProviderPtr RouteConfigProviderUtil::create( factory_context); case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds: return route_config_provider_manager.createRdsRouteConfigProvider( - config.rds(), factory_context, stat_prefix, factory_context.initManager()); + config.rds(), factory_context, stat_prefix, factory_context.initManager(), is_delta); default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -56,7 +56,7 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Envoy::Router::RouteConfigProviderManagerImpl& route_config_provider_manager) + Envoy::Router::RouteConfigProviderManagerImpl& route_config_provider_manager, bool is_delta) : route_config_name_(rds.route_config_name()), factory_context_(factory_context), init_target_(fmt::format("RdsRouteConfigSubscription {}", route_config_name_), [this]() { subscription_->start({route_config_name_}); }), @@ -69,7 +69,7 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( rds.config_source(), Grpc::Common::typeUrl(envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name()), - *scope_, *this); + *scope_, *this, is_delta); config_update_info_ = std::make_unique( factory_context.timeSource(), factory_context.messageValidationVisitor()); @@ -225,7 +225,7 @@ RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(Server::Admin& ad Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Init::Manager& init_manager) { + Init::Manager& init_manager, bool is_delta) { // RdsRouteConfigSubscriptions are unique based on their serialized RDS config. const uint64_t manager_identifier = MessageUtil::hash(rds); @@ -237,7 +237,7 @@ Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteCon // around it. However, since this is not a performance critical path we err on the side // of simplicity. subscription.reset(new RdsRouteConfigSubscription(rds, manager_identifier, factory_context, - stat_prefix, *this)); + stat_prefix, *this, is_delta)); init_manager.add(subscription->init_target_); route_config_subscriptions_.insert({manager_identifier, subscription}); } else { diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 418512bb6e50..caf2340f084e 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -48,7 +48,7 @@ class RouteConfigProviderUtil { create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - RouteConfigProviderManager& route_config_provider_manager); + RouteConfigProviderManager& route_config_provider_manager, bool is_delta); }; class RouteConfigProviderManagerImpl; @@ -132,8 +132,8 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, RdsRouteConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix, - RouteConfigProviderManagerImpl& route_config_provider_manager); + const std::string& stat_prefix, RouteConfigProviderManagerImpl& route_config_provider_manager, + bool is_delta); bool validateUpdateSize(int num_resources); @@ -207,7 +207,7 @@ class RouteConfigProviderManagerImpl : public RouteConfigProviderManager, RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Init::Manager& init_manager) override; + Init::Manager& init_manager, bool is_delta) override; RouteConfigProviderPtr createStaticRouteConfigProvider(const envoy::api::v2::RouteConfiguration& route_config, diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 9c710540930f..d7ae71b134bc 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -98,7 +98,7 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( scoped_rds.scoped_rds_config_source(), Grpc::Common::typeUrl( envoy::api::v2::ScopedRouteConfiguration().GetDescriptor()->full_name()), - *scope_, *this); + *scope_, *this, true); initialize([scope_key_builder]() -> Envoy::Config::ConfigProvider::ConfigConstSharedPtr { return std::make_shared( @@ -115,7 +115,7 @@ ScopedRdsConfigSubscription::RdsRouteConfigProviderHelper::RdsRouteConfigProvide route_provider_(static_cast( parent_.route_config_provider_manager_ .createRdsRouteConfigProvider(rds, parent_.factory_context_, parent_.stat_prefix_, - init_manager) + init_manager, true) .release())), rds_update_callback_handle_(route_provider_->subscription().addUpdateCallback([this]() { // Subscribe to RDS update. @@ -221,8 +221,10 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. // In the case if factory_context_.initManager() is uninitialized, RDS is already paused either // by Server init or LDS init. - factory_context_.clusterManager().adsMux().pause( - Envoy::Config::TypeUrl::get().RouteConfiguration); + if (factory_context_.clusterManager().adsMux()) { + factory_context_.clusterManager().adsMux()->pause( + Envoy::Config::TypeUrl::get().RouteConfiguration); + } resume_rds = std::make_unique([this, &noop_init_manager, version_info] { // For new RDS subscriptions created after listener warming up, we don't wait for them to warm // up. @@ -234,8 +236,10 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // New RDS subscriptions should have been created, now lift the floodgate. // Note in the case of partial acceptance, accepted RDS subscriptions should be started // despite of any error. - factory_context_.clusterManager().adsMux().resume( - Envoy::Config::TypeUrl::get().RouteConfiguration); + if (factory_context_.clusterManager().adsMux()) { + factory_context_.clusterManager().adsMux()->resume( + Envoy::Config::TypeUrl::get().RouteConfiguration); + } }); } std::vector exception_msgs; diff --git a/source/common/router/vhds.cc b/source/common/router/vhds.cc index 93d304647f7d..de67108a9e26 100644 --- a/source/common/router/vhds.cc +++ b/source/common/router/vhds.cc @@ -23,11 +23,11 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, const std::string& stat_prefix, std::unordered_set& route_config_providers) : config_update_info_(config_update_info), - init_target_(fmt::format("VhdsConfigSubscription {}", config_update_info_->routeConfigName()), - [this]() { subscription_->start({}); }), scope_(factory_context.scope().createScope(stat_prefix + "vhds." + config_update_info_->routeConfigName() + ".")), stats_({ALL_VHDS_STATS(POOL_COUNTER(*scope_))}), + init_target_(fmt::format("VhdsConfigSubscription {}", config_update_info_->routeConfigName()), + [this]() { subscription_->start({}); }), route_config_providers_(route_config_providers) { const auto& config_source = config_update_info_->routeConfiguration() .vhds() @@ -42,7 +42,7 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( config_update_info_->routeConfiguration().vhds().config_source(), Grpc::Common::typeUrl(envoy::api::v2::route::VirtualHost().GetDescriptor()->full_name()), - *scope_, *this); + *scope_, *this, /*is_delta=*/true); } void VhdsSubscription::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, diff --git a/source/common/router/vhds.h b/source/common/router/vhds.h index a2d3cb6beb81..639f97dd76f1 100644 --- a/source/common/router/vhds.h +++ b/source/common/router/vhds.h @@ -63,10 +63,10 @@ class VhdsSubscription : Envoy::Config::SubscriptionCallbacks, } RouteConfigUpdatePtr& config_update_info_; - std::unique_ptr subscription_; - Init::TargetImpl init_target_; Stats::ScopePtr scope_; VhdsStats stats_; + std::unique_ptr subscription_; + Init::TargetImpl init_target_; std::unordered_set& route_config_providers_; }; diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 3d0a721c6aa1..22a0508bc56a 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -568,7 +568,7 @@ void RtdsSubscription::start() { subscription_ = parent_.cm_->subscriptionFactory().subscriptionFromConfigSource( config_source_, Grpc::Common::typeUrl(envoy::service::discovery::v2::Runtime().GetDescriptor()->full_name()), - store_, *this); + store_, *this, /*is_delta=*/false); subscription_->start({resource_name_}); } diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index 4514f5ce0792..3e3478f0beb7 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -79,7 +79,7 @@ void SdsApi::initialize() { subscription_ = subscription_factory_.subscriptionFromConfigSource( sds_config_, Grpc::Common::typeUrl(envoy::api::v2::auth::Secret().GetDescriptor()->full_name()), stats_, - *this); + *this, /*is_delta=*/false); subscription_->start({sds_config_name_}); } diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index 838ec09721b7..3269d58a2c3f 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -15,19 +15,22 @@ namespace Envoy { namespace Upstream { -CdsApiPtr CdsApiImpl::create(const envoy::api::v2::core::ConfigSource& cds_config, +// TODO(fredlas) the is_delta argument can be removed upon delta+SotW ADS Envoy code unification. It +// is only actually needed to choose the grpc_method, which is irrelevant if ADS is used. +CdsApiPtr CdsApiImpl::create(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, ClusterManager& cm, Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validation_visitor) { - return CdsApiPtr{new CdsApiImpl(cds_config, cm, scope, validation_visitor)}; + return CdsApiPtr{new CdsApiImpl(cds_config, is_delta, cm, scope, validation_visitor)}; } -CdsApiImpl::CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, - Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validation_visitor) +CdsApiImpl::CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, + ClusterManager& cm, Stats::Scope& scope, + ProtobufMessage::ValidationVisitor& validation_visitor) : cm_(cm), scope_(scope.createScope("cluster_manager.cds.")), validation_visitor_(validation_visitor) { subscription_ = cm_.subscriptionFactory().subscriptionFromConfigSource( cds_config, Grpc::Common::typeUrl(envoy::api::v2::Cluster().GetDescriptor()->full_name()), - *scope_, *this); + *scope_, *this, is_delta); } void CdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, @@ -56,8 +59,12 @@ void CdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - cm_.adsMux().pause(Config::TypeUrl::get().ClusterLoadAssignment); - Cleanup eds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + std::unique_ptr maybe_eds_resume; + if (cm_.adsMux()) { + cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); + maybe_eds_resume = std::make_unique( + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + } ENVOY_LOG(info, "cds: add {} cluster(s), remove {} cluster(s)", added_resources.size(), removed_resources.size()); diff --git a/source/common/upstream/cds_api_impl.h b/source/common/upstream/cds_api_impl.h index b17d4bbc9989..66bb7e4c3df8 100644 --- a/source/common/upstream/cds_api_impl.h +++ b/source/common/upstream/cds_api_impl.h @@ -22,8 +22,8 @@ class CdsApiImpl : public CdsApi, Config::SubscriptionCallbacks, Logger::Loggable { public: - static CdsApiPtr create(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, - Stats::Scope& scope, + static CdsApiPtr create(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, + ClusterManager& cm, Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validation_visitor); // Upstream::CdsApi @@ -46,8 +46,9 @@ class CdsApiImpl : public CdsApi, return MessageUtil::anyConvert(resource).name(); } - CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, - Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validation_visitor); + CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, + ClusterManager& cm, Stats::Scope& scope, + ProtobufMessage::ValidationVisitor& validation_visitor); void runInitializeCallbackIfAny(); ClusterManager& cm_; diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 70b4c9eb9c01..1d57d737ad44 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -18,6 +18,7 @@ #include "common/common/enum_to_int.h" #include "common/common/fmt.h" #include "common/common/utility.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/resources.h" #include "common/config/utility.h" #include "common/grpc/async_client_manager_impl.h" @@ -137,16 +138,16 @@ void ClusterManagerInitHelper::maybeFinishInitialize() { // If the first CDS response doesn't have any primary cluster, ClusterLoadAssignment // should be already paused by CdsApiImpl::onConfigUpdate(). Need to check that to // avoid double pause ClusterLoadAssignment. - if (cm_.adsMux().paused(Config::TypeUrl::get().ClusterLoadAssignment)) { + if (cm_.adsMux() == nullptr || + cm_.adsMux()->paused(Config::TypeUrl::get().ClusterLoadAssignment)) { initializeSecondaryClusters(); } else { - cm_.adsMux().pause(Config::TypeUrl::get().ClusterLoadAssignment); + cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); Cleanup eds_resume( - [this] { cm_.adsMux().resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); initializeSecondaryClusters(); } } - return; } @@ -220,6 +221,33 @@ ClusterManagerImpl::ClusterManagerImpl( } } + // TODO(fredlas) HACK to support + // loadCluster->clusterFromProto->ClusterFactoryImplBase::create->EdsClusterFactory::createClusterImpl(), + // which wants to call xdsIsDelta() on us. So, we need to get our xds_is_delta_ defined before + // then. Once SotW and delta are unified, that is_delta bool will be gone from everywhere, and the + // xds_is_delta_ variable can be removed. + const auto& dyn_resources = bootstrap.dynamic_resources(); + if (dyn_resources.has_ads_config()) { + xds_is_delta_ = + dyn_resources.ads_config().api_type() == envoy::api::v2::core::ApiConfigSource::DELTA_GRPC; + } else if (dyn_resources.has_cds_config()) { + const auto& cds_config = dyn_resources.cds_config(); + xds_is_delta_ = + cds_config.api_config_source().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC || + (dyn_resources.has_ads_config() && dyn_resources.ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); + } else if (dyn_resources.has_lds_config()) { + const auto& lds_config = dyn_resources.lds_config(); + xds_is_delta_ = + lds_config.api_config_source().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC || + (dyn_resources.has_ads_config() && dyn_resources.ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); + } else { + xds_is_delta_ = false; + } + // Cluster loading happens in two phases: first all the primary clusters are loaded, and then all // the secondary clusters are loaded. As it currently stands all non-EDS clusters are primary and // only EDS clusters are secondary. This two phase loading is done because in v2 configuration @@ -233,18 +261,37 @@ ClusterManagerImpl::ClusterManagerImpl( } // Now setup ADS if needed, this might rely on a primary cluster. - if (bootstrap.dynamic_resources().has_ads_config()) { - ads_mux_ = std::make_unique( - local_info, - Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, bootstrap.dynamic_resources().ads_config(), stats) - ->create(), - main_thread_dispatcher, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, - Envoy::Config::Utility::parseRateLimitSettings(bootstrap.dynamic_resources().ads_config()), - bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + // This is the only point where distinction between delta ADS and state-of-the-world ADS is made. + // After here, we just have a GrpcMux interface held in ads_mux_, which hides + // whether the backing implementation is delta or SotW. + if (dyn_resources.has_ads_config()) { + if (dyn_resources.ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC) { + auto& api_config_source = dyn_resources.has_ads_config() + ? dyn_resources.ads_config() + : dyn_resources.cds_config().api_config_source(); + ads_mux_ = std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, api_config_source, + stats) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.DeltaAggregatedResources"), + random_, stats_, + Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info); + } else { + ads_mux_ = std::make_shared( + local_info, + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, + dyn_resources.ads_config(), stats) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + random_, stats_, + Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), + bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + } } else { ads_mux_ = std::make_unique(); } @@ -278,8 +325,8 @@ ClusterManagerImpl::ClusterManagerImpl( }); // We can now potentially create the CDS API once the backing cluster exists. - if (bootstrap.dynamic_resources().has_cds_config()) { - cds_api_ = factory_.createCds(bootstrap.dynamic_resources().cds_config(), *this); + if (dyn_resources.has_cds_config()) { + cds_api_ = factory_.createCds(dyn_resources.cds_config(), xds_is_delta_, *this); init_helper_.setCds(cds_api_.get()); } else { init_helper_.setCds(nullptr); @@ -1311,9 +1358,10 @@ std::pair ProdClusterManagerFactor } CdsApiPtr ProdClusterManagerFactory::createCds(const envoy::api::v2::core::ConfigSource& cds_config, - ClusterManager& cm) { + bool is_delta, ClusterManager& cm) { // TODO(htuch): Differentiate static vs. dynamic validation visitors. - return CdsApiImpl::create(cds_config, cm, stats_, validation_context_.dynamicValidationVisitor()); + return CdsApiImpl::create(cds_config, is_delta, cm, stats_, + validation_context_.dynamicValidationVisitor()); } } // namespace Upstream diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 4838178804d6..7c969984ca7c 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -68,7 +68,7 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { std::pair clusterFromProto(const envoy::api::v2::Cluster& cluster, ClusterManager& cm, Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api) override; - CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, + CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, ClusterManager& cm) override; Secret::SecretManager& secretManager() override { return secret_manager_; } @@ -224,7 +224,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggablefull_name()), - info_->statsScope(), *this); + info_->statsScope(), *this, is_delta); } void EdsClusterImpl::startPreInit() { subscription_->start({cluster_name_}); } @@ -268,8 +268,9 @@ EdsClusterFactory::createClusterImpl( } return std::make_pair( - std::make_shared(cluster, context.runtime(), socket_factory_context, - std::move(stats_scope), context.addedViaApi()), + std::make_unique(cluster, context.runtime(), socket_factory_context, + std::move(stats_scope), context.addedViaApi(), + context.clusterManager().xdsIsDelta()), nullptr); } diff --git a/source/common/upstream/eds.h b/source/common/upstream/eds.h index 0df5f4c84473..3f0a74b8dfca 100644 --- a/source/common/upstream/eds.h +++ b/source/common/upstream/eds.h @@ -24,7 +24,7 @@ class EdsClusterImpl : public BaseDynamicClusterImpl, Config::SubscriptionCallba public: EdsClusterImpl(const envoy::api::v2::Cluster& cluster, Runtime::Loader& runtime, Server::Configuration::TransportSocketFactoryContext& factory_context, - Stats::ScopePtr&& stats_scope, bool added_via_api); + Stats::ScopePtr&& stats_scope, bool added_via_api, bool is_delta); // Upstream::Cluster InitializePhase initializePhase() const override { return InitializePhase::Secondary; } diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 998dce6709ab..b18316ad5217 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -183,7 +183,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kRouteConfig: route_config_provider_ = Router::RouteConfigProviderUtil::create( - config, context_, stats_prefix_, route_config_provider_manager_); + config, context_, stats_prefix_, route_config_provider_manager_, + context_.clusterManager().xdsIsDelta()); break; case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kScopedRoutes: diff --git a/source/server/config_validation/cluster_manager.cc b/source/server/config_validation/cluster_manager.cc index cf20a6c0e221..0f014a1741ea 100644 --- a/source/server/config_validation/cluster_manager.cc +++ b/source/server/config_validation/cluster_manager.cc @@ -14,9 +14,9 @@ ClusterManagerPtr ValidationClusterManagerFactory::clusterManagerFromProto( CdsApiPtr ValidationClusterManagerFactory::createCds(const envoy::api::v2::core::ConfigSource& cds_config, - ClusterManager& cm) { + bool is_delta, ClusterManager& cm) { // Create the CdsApiImpl... - ProdClusterManagerFactory::createCds(cds_config, cm); + ProdClusterManagerFactory::createCds(cds_config, is_delta, cm); // ... and then throw it away, so that we don't actually connect to it. return nullptr; } diff --git a/source/server/config_validation/cluster_manager.h b/source/server/config_validation/cluster_manager.h index 0fe33294d190..b62387c3341d 100644 --- a/source/server/config_validation/cluster_manager.h +++ b/source/server/config_validation/cluster_manager.h @@ -38,7 +38,7 @@ class ValidationClusterManagerFactory : public ProdClusterManagerFactory { // Delegates to ProdClusterManagerFactory::createCds, but discards the result and returns nullptr // unconditionally. - CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, + CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, ClusterManager& cm) override; private: diff --git a/source/server/config_validation/server.h b/source/server/config_validation/server.h index d447c5822940..58ad9affadc3 100644 --- a/source/server/config_validation/server.h +++ b/source/server/config_validation/server.h @@ -109,10 +109,11 @@ class ValidationInstance : Logger::Loggable, } // Server::ListenerComponentFactory - LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { - return std::make_unique(lds_config, clusterManager(), initManager(), stats(), - listenerManager(), - messageValidationContext().dynamicValidationVisitor()); + LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) override { + return std::make_unique( + lds_config, clusterManager(), initManager(), stats(), listenerManager(), + messageValidationContext().dynamicValidationVisitor(), is_delta); } std::vector createNetworkFilterFactoryList( const Protobuf::RepeatedPtrField& filters, diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index d0cce626afb1..ca5cb3ed187d 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -17,13 +17,13 @@ namespace Server { LdsApiImpl::LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, Upstream::ClusterManager& cm, Init::Manager& init_manager, Stats::Scope& scope, ListenerManager& lm, - ProtobufMessage::ValidationVisitor& validation_visitor) + ProtobufMessage::ValidationVisitor& validation_visitor, bool is_delta) : listener_manager_(lm), scope_(scope.createScope("listener_manager.lds.")), cm_(cm), init_target_("LDS", [this]() { subscription_->start({}); }), validation_visitor_(validation_visitor) { subscription_ = cm.subscriptionFactory().subscriptionFromConfigSource( lds_config, Grpc::Common::typeUrl(envoy::api::v2::Listener().GetDescriptor()->full_name()), - *scope_, *this); + *scope_, *this, is_delta); init_manager.add(init_target_); } @@ -31,8 +31,12 @@ void LdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - cm_.adsMux().pause(Config::TypeUrl::get().RouteConfiguration); - Cleanup rds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().RouteConfiguration); }); + std::unique_ptr maybe_eds_resume; + if (cm_.adsMux()) { + cm_.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); + maybe_eds_resume = std::make_unique( + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); + } bool any_applied = false; // We do all listener removals before adding the new listeners. This allows adding a new listener diff --git a/source/server/lds_api.h b/source/server/lds_api.h index 24ca66cfe73f..2ef9209542b2 100644 --- a/source/server/lds_api.h +++ b/source/server/lds_api.h @@ -24,7 +24,7 @@ class LdsApiImpl : public LdsApi, public: LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, Upstream::ClusterManager& cm, Init::Manager& init_manager, Stats::Scope& scope, ListenerManager& lm, - ProtobufMessage::ValidationVisitor& validation_visitor); + ProtobufMessage::ValidationVisitor& validation_visitor, bool is_delta); // Server::LdsApi std::string versionInfo() const override { return system_version_info_; } diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index 599f471ea7fe..ca00bc11082e 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -59,10 +59,12 @@ class ProdListenerComponentFactory : public ListenerComponentFactory, Configuration::ListenerFactoryContext& context); // Server::ListenerComponentFactory - LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { + LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) override { return std::make_unique( lds_config, server_.clusterManager(), server_.initManager(), server_.stats(), - server_.listenerManager(), server_.messageValidationContext().dynamicValidationVisitor()); + server_.listenerManager(), server_.messageValidationContext().dynamicValidationVisitor(), + is_delta); } std::vector createNetworkFilterFactoryList( const Protobuf::RepeatedPtrField& filters, @@ -127,9 +129,9 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable> listeners() override; uint64_t numConnections() override; diff --git a/source/server/server.cc b/source/server/server.cc index 9653cdc8e792..1afeff023cfe 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -402,7 +402,13 @@ void InstanceImpl::initialize(const Options& options, // Instruct the listener manager to create the LDS provider if needed. This must be done later // because various items do not yet exist when the listener manager is created. if (bootstrap_.dynamic_resources().has_lds_config()) { - listener_manager_->createLdsApi(bootstrap_.dynamic_resources().lds_config()); + const bool is_delta = + bootstrap_.dynamic_resources().lds_config().api_config_source().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC || + (bootstrap_.dynamic_resources().has_ads_config() && + bootstrap_.dynamic_resources().ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); + listener_manager_->createLdsApi(bootstrap_.dynamic_resources().lds_config(), is_delta); } // We have to defer RTDS initialization until after the cluster manager is @@ -516,14 +522,18 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch // Pause RDS to ensure that we don't send any requests until we've // subscribed to all the RDS resources. The subscriptions happen in the init callbacks, // so we pause RDS until we've completed all the callbacks. - cm.adsMux().pause(Config::TypeUrl::get().RouteConfiguration); + if (cm.adsMux()) { + cm.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); + } ENVOY_LOG(info, "all clusters initialized. initializing init manager"); init_manager.initialize(init_watcher_); // Now that we're execute all the init callbacks we can resume RDS // as we've subscribed to all the statically defined RDS resources. - cm.adsMux().resume(Config::TypeUrl::get().RouteConfiguration); + if (cm.adsMux()) { + cm.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); + } }); } diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 60101fa4aa22..ad8eff35f154 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -95,6 +95,30 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "new_grpc_mux_impl_test", + srcs = ["new_grpc_mux_impl_test.cc"], + deps = [ + "//source/common/config:new_grpc_mux_lib", + "//source/common/config:protobuf_link_hacks", + "//source/common/config:resources_lib", + "//source/common/protobuf", + "//source/common/stats:isolated_store_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:discovery_cc", + "@envoy_api//envoy/api/v2:eds_cc", + "@envoy_api//envoy/service/discovery/v2:ads_cc", + ], +) + envoy_cc_test( name = "grpc_stream_test", srcs = ["grpc_stream_test.cc"], diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 26d7daca0df3..f8127f93cfdf 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -9,20 +9,25 @@ namespace { class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public testing::Test { protected: DeltaSubscriptionImplTest() = default; + + // We need to destroy the subscription before the test's destruction, because the subscription's + // destructor removes its watch from the NewGrpcMuxImpl, and that removal process involves + // some things held by the test fixture. + void TearDown() override { doSubscriptionTearDown(); } }; TEST_F(DeltaSubscriptionImplTest, UpdateResourcesCausesRequest) { startSubscription({"name1", "name2", "name3"}); expectSendMessage({"name4"}, {"name1", "name2"}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); expectSendMessage({"name1", "name2"}, {}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); expectSendMessage({}, {"name1", "name2"}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); expectSendMessage({"name1", "name2"}, {}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); expectSendMessage({}, {"name1", "name2", "name3"}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name4"}); + subscription_->updateResourceInterest({"name4"}); } // Checks that after a pause(), no requests are sent until resume(). @@ -36,11 +41,11 @@ TEST_F(DeltaSubscriptionImplTest, PauseHoldsRequest) { expectSendMessage({"name4"}, {"name1", "name2"}, Grpc::Status::GrpcStatus::Ok, "", {}); // If not for the pause, these updates would make the expectSendMessage fail due to too many // messages being sent. - subscription_->updateResources({"name3", "name4"}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); - subscription_->updateResources({"name3", "name4"}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); - subscription_->updateResources({"name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); subscription_->resume(); } @@ -64,8 +69,10 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { resource->set_version("version1A"); const std::string nonce = std::to_string(HashUtil::xxHash64("version1A")); message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - subscription_->onDiscoveryResponse(std::move(message)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(message)); } // The server gives us our first version of resource name2. // subscription_ now wants to ACK name1 and then name2 (but can't due to pause). @@ -76,8 +83,10 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { resource->set_version("version2A"); const std::string nonce = std::to_string(HashUtil::xxHash64("version2A")); message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - subscription_->onDiscoveryResponse(std::move(message)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(message)); } // The server gives us an updated version of resource name1. // subscription_ now wants to ACK name1A, then name2, then name1B (but can't due to pause). @@ -88,8 +97,10 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { resource->set_version("version1B"); const std::string nonce = std::to_string(HashUtil::xxHash64("version1B")); message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - subscription_->onDiscoveryResponse(std::move(message)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(message)); } // All ACK sendMessage()s will happen upon calling resume(). EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)) @@ -106,13 +117,36 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { // in the correct order. } -TEST_F(DeltaSubscriptionImplTest, NoGrpcStream) { - // Have to call start() to get state_ populated (which this test needs to not segfault), but - // start() also tries to start the GrpcStream. So, have that attempt return nullptr. - EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)).Times(0); - subscription_->start({"name1"}); - subscription_->updateResources({"name1", "name2"}); +TEST(DeltaSubscriptionImplFixturelessTest, NoGrpcStream) { + Stats::IsolatedStoreImpl stats_store; + SubscriptionStats stats(Utility::generateStats(stats_store)); + + envoy::api::v2::core::Node node; + node.set_id("fo0"); + NiceMock local_info; + EXPECT_CALL(local_info, node()).WillRepeatedly(testing::ReturnRef(node)); + + NiceMock dispatcher; + NiceMock random; + Envoy::Config::RateLimitSettings rate_limit_settings; + NiceMock> callbacks; + auto* async_client = new Grpc::MockAsyncClient(); + + const Protobuf::MethodDescriptor* method_descriptor = + Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints"); + std::shared_ptr xds_context = std::make_shared( + std::unique_ptr(async_client), dispatcher, *method_descriptor, random, + stats_store, rate_limit_settings, local_info); + + std::unique_ptr subscription = std::make_unique( + xds_context, Config::TypeUrl::get().ClusterLoadAssignment, callbacks, stats, + std::chrono::milliseconds(12345), false); + + EXPECT_CALL(*async_client, startRaw(_, _, _)).WillOnce(Return(nullptr)); + + subscription->start({"name1"}); + subscription->updateResourceInterest({"name1", "name2"}); } } // namespace diff --git a/test/common/config/delta_subscription_state_test.cc b/test/common/config/delta_subscription_state_test.cc index 462a91e2184d..2a96f2b0953f 100644 --- a/test/common/config/delta_subscription_state_test.cc +++ b/test/common/config/delta_subscription_state_test.cc @@ -22,10 +22,9 @@ const char TypeUrl[] = "type.googleapis.com/envoy.api.v2.Cluster"; class DeltaSubscriptionStateTest : public testing::Test { protected: DeltaSubscriptionStateTest() - : stats_(Utility::generateStats(store_)), - state_(TypeUrl, {"name1", "name2", "name3"}, callbacks_, local_info_, - std::chrono::milliseconds(0U), dispatcher_, stats_) { - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + : state_(TypeUrl, callbacks_, local_info_, std::chrono::milliseconds(0U), dispatcher_) { + state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name1", "name2", "name3")); } @@ -62,8 +61,6 @@ class DeltaSubscriptionStateTest : public testing::Test { NiceMock> callbacks_; NiceMock local_info_; NiceMock dispatcher_; - Stats::IsolatedStoreImpl store_; - SubscriptionStats stats_; // We start out interested in three resources: name1, name2, and name3. DeltaSubscriptionState state_; }; @@ -82,14 +79,14 @@ populateRepeatedResource(std::vector> items) // Basic gaining/losing interest in resources should lead to (un)subscriptions. TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { { - state_.updateResourceInterest({"name2", "name3", "name4"}); // drop name1, add name4 - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {"name1"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1")); } { - state_.updateResourceInterest({"name1", "name2"}); // add back name1, drop name3 and 4 - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name1"}, {"name3", "name4"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name1")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name3", "name4")); } @@ -105,9 +102,9 @@ TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { // interpret the resource_names_subscribe field as "send these resources even if you think Envoy // already has them". TEST_F(DeltaSubscriptionStateTest, RemoveThenAdd) { - state_.updateResourceInterest({"name1", "name2"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({}, {"name3"}); + state_.updateSubscriptionInterest({"name3"}, {}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name3")); EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } @@ -121,29 +118,29 @@ TEST_F(DeltaSubscriptionStateTest, RemoveThenAdd) { // should be like this. What *is* important: the server must happily and cleanly ignore // "unsubscribe from [resource name I have never before referred to]" requests. TEST_F(DeltaSubscriptionStateTest, AddThenRemove) { - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({}, {"name4"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_TRUE(cur_request.resource_names_subscribe().empty()); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name4")); } // add/remove/add == add. TEST_F(DeltaSubscriptionStateTest, AddRemoveAdd) { - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({}, {"name4"}); + state_.updateSubscriptionInterest({"name4"}, {}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } // remove/add/remove == remove. TEST_F(DeltaSubscriptionStateTest, RemoveAddRemove) { - state_.updateResourceInterest({"name1", "name2"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - state_.updateResourceInterest({"name1", "name2"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({}, {"name3"}); + state_.updateSubscriptionInterest({"name3"}, {}); + state_.updateSubscriptionInterest({}, {"name3"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_TRUE(cur_request.resource_names_subscribe().empty()); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name3")); } @@ -151,19 +148,19 @@ TEST_F(DeltaSubscriptionStateTest, RemoveAddRemove) { // Starts with 1,2,3. 4 is added/removed/added. In those same updates, 1,2,3 are // removed/added/removed. End result should be 4 added and 1,2,3 removed. TEST_F(DeltaSubscriptionStateTest, BothAddAndRemove) { - state_.updateResourceInterest({"name4"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - state_.updateResourceInterest({"name4"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); + state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {"name4"}); + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2", "name3")); } TEST_F(DeltaSubscriptionStateTest, CumulativeUpdates) { - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); - state_.updateResourceInterest({"name1", "name2", "name3", "name4", "name5"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({"name5"}, {}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4", "name5")); EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } @@ -217,7 +214,7 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); deliverDiscoveryResponse(add1_2, {}, "debugversion1"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); EXPECT_EQ(cur_request.initial_resource_versions().end(), @@ -232,7 +229,7 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove2.Add() = "name2"; deliverDiscoveryResponse(add1_3, remove2, "debugversion2"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_EQ("version1B", cur_request.initial_resource_versions().at("name1")); EXPECT_EQ(cur_request.initial_resource_versions().end(), cur_request.initial_resource_versions().find("name2")); @@ -246,15 +243,15 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove1_3.Add() = "name3"; deliverDiscoveryResponse({}, remove1_3, "debugversion3"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_TRUE(cur_request.initial_resource_versions().empty()); } { // ...but our own map should remember our interest. In particular, losing interest in a // resource should cause its name to appear in the next request's resource_names_unsubscribe. - state_.updateResourceInterest({"name3", "name4"}); // note the lack of 1 and 2 - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2")); } @@ -273,9 +270,9 @@ TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribeAfterReconnect) { populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); deliverDiscoveryResponse(add1_2, {}, "debugversion1"); - state_.updateResourceInterest({"name2", "name3", "name4"}); // drop name1, add name4 - state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {"name1"}); + state_.markStreamFresh(); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); // Regarding the resource_names_subscribe field: // name1: do not include: we lost interest. // name2: yes do include: we're interested and we have a version of it. @@ -296,7 +293,7 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { {{"name1", "version1A"}, {"name2", "version2A"}, {"name3", "version3A"}}); deliverDiscoveryResponse(add_all, {}, "debugversion1"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); EXPECT_EQ("version3A", cur_request.initial_resource_versions().at("name3")); @@ -304,7 +301,7 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { // Then, after updating the resources but not reconnecting the stream, verify that initial // versions are not sent. { - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); + state_.updateSubscriptionInterest({"name4"}, {}); // The xDS server updates our resources, and gives us our newly requested one too. Protobuf::RepeatedPtrField add_all = populateRepeatedResource({{"name1", "version1B"}, @@ -312,7 +309,7 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { {"name3", "version3B"}, {"name4", "version4A"}}); deliverDiscoveryResponse(add_all, {}, "debugversion2"); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_TRUE(cur_request.initial_resource_versions().empty()); } } @@ -321,28 +318,16 @@ TEST_F(DeltaSubscriptionStateTest, CheckUpdatePending) { // Note that the test fixture ctor causes the first request to be "sent", so we start in the // middle of a stream, with our initially interested resources having been requested already. EXPECT_FALSE(state_.subscriptionUpdatePending()); - state_.updateResourceInterest({"name1", "name2", "name3"}); // no change + state_.updateSubscriptionInterest({}, {}); // no change EXPECT_FALSE(state_.subscriptionUpdatePending()); state_.markStreamFresh(); - EXPECT_TRUE(state_.subscriptionUpdatePending()); // no change, BUT fresh stream - state_.updateResourceInterest({"name1", "name2"}); // one removed + EXPECT_TRUE(state_.subscriptionUpdatePending()); // no change, BUT fresh stream + state_.updateSubscriptionInterest({}, {"name3"}); // one removed EXPECT_TRUE(state_.subscriptionUpdatePending()); - state_.updateResourceInterest({"name1", "name2", "name3"}); // one added + state_.updateSubscriptionInterest({"name3"}, {}); // one added EXPECT_TRUE(state_.subscriptionUpdatePending()); } -TEST_F(DeltaSubscriptionStateTest, PauseAndResume) { - EXPECT_FALSE(state_.paused()); - state_.pause(); - EXPECT_TRUE(state_.paused()); - state_.resume(); - EXPECT_FALSE(state_.paused()); - state_.pause(); - EXPECT_TRUE(state_.paused()); - state_.resume(); - EXPECT_FALSE(state_.paused()); -} - // The next three tests test that duplicate resource names (whether additions or removals) cause // DeltaSubscriptionState to reject the update without even trying to hand it to the consuming API's // onConfigUpdate(). @@ -374,14 +359,6 @@ TEST_F(DeltaSubscriptionStateTest, AddedAndRemoved) { ack.error_detail_.message()); } -TEST_F(DeltaSubscriptionStateTest, handleEstablishmentFailure) { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)).Times(0); - - state_.handleEstablishmentFailure(); - EXPECT_EQ(stats_.update_failure_.value(), 1); - EXPECT_EQ(stats_.update_attempt_.value(), 1); -} - } // namespace } // namespace Config } // namespace Envoy diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 4414aa77489c..35e8a4bbef06 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "common/config/delta_subscription_impl.h" #include "common/grpc/common.h" @@ -32,15 +34,30 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { node_.set_id("fo0"); EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)); + xds_context_ = std::make_shared( + std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, + random_, stats_store_, rate_limit_settings_, local_info_); subscription_ = std::make_unique( - local_info_, std::unique_ptr(async_client_), dispatcher_, - *method_descriptor_, Config::TypeUrl::get().ClusterLoadAssignment, random_, stats_store_, - rate_limit_settings_, callbacks_, stats_, init_fetch_timeout); + xds_context_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, + init_fetch_timeout, false); + EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); + } + + void doSubscriptionTearDown() override { + if (subscription_started_) { + EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)); + subscription_.reset(); + } } ~DeltaSubscriptionTestHarness() override { while (!nonce_acks_required_.empty()) { - EXPECT_FALSE(nonce_acks_sent_.empty()); + if (nonce_acks_sent_.empty()) { + // It's not enough to EXPECT_FALSE(nonce_acks_sent_.empty()), we need to skip the following + // EXPECT_EQ, otherwise the undefined .front() can get pretty bad. + EXPECT_FALSE(nonce_acks_sent_.empty()); + break; + } EXPECT_EQ(nonce_acks_required_.front(), nonce_acks_sent_.front()); nonce_acks_required_.pop(); nonce_acks_sent_.pop(); @@ -49,7 +66,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { } void startSubscription(const std::set& cluster_names) override { - EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); + subscription_started_ = true; last_cluster_names_ = cluster_names; expectSendMessage(last_cluster_names_, ""); subscription_->start(cluster_names); @@ -104,12 +121,11 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { void deliverConfigUpdate(const std::vector& cluster_names, const std::string& version, bool accept) override { - std::unique_ptr response( - new envoy::api::v2::DeltaDiscoveryResponse()); - + auto response = std::make_unique(); last_response_nonce_ = std::to_string(HashUtil::xxHash64(version)); response->set_nonce(last_response_nonce_); response->set_system_version_info(version); + response->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); Protobuf::RepeatedPtrField typed_resources; for (const auto& cluster : cluster_names) { @@ -132,11 +148,12 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config", {}); } - subscription_->onDiscoveryResponse(std::move(response)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } - void updateResources(const std::set& cluster_names) override { + void updateResourceInterest(const std::set& cluster_names) override { std::set sub; std::set unsub; @@ -147,7 +164,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { std::inserter(unsub, unsub.begin())); expectSendMessage(sub, unsub, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources(cluster_names); + subscription_->updateResourceInterest(cluster_names); last_cluster_names_ = cluster_names; } @@ -157,7 +174,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { init_timeout_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout), _)); + EXPECT_CALL(*init_timeout_timer_, enableTimer(timeout, _)); } void expectDisableInitFetchTimeoutTimer() override { @@ -172,6 +189,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock random_; NiceMock local_info_; Grpc::MockAsyncStream async_stream_; + std::shared_ptr xds_context_; std::unique_ptr subscription_; std::string last_response_nonce_; std::set last_cluster_names_; @@ -181,6 +199,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock> callbacks_; std::queue nonce_acks_required_; std::queue nonce_acks_sent_; + bool subscription_started_{}; }; } // namespace diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 901b5cf29b31..1958a4270bd6 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -43,11 +43,11 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { subscription_.start(cluster_names); } - void updateResources(const std::set& cluster_names) override { - subscription_.updateResources(cluster_names); + void updateResourceInterest(const std::set& cluster_names) override { + subscription_.updateResourceInterest(cluster_names); } - void updateFile(const std::string json, bool run_dispatcher = true) { + void updateFile(const std::string& json, bool run_dispatcher = true) { // Write JSON contents to file, rename to path_ and run dispatcher to catch // inotify. const std::string temp_path = TestEnvironment::writeStringToFileForTest("eds.json.tmp", json); @@ -94,13 +94,10 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { version); } - void expectConfigUpdateFailed() override { - // initial_fetch_timeout not implemented - } + void expectConfigUpdateFailed() override { stats_.update_failure_.inc(); } - void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { - UNREFERENCED_PARAMETER(timeout); - // initial_fetch_timeout not implemented + void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds) override { + // initial_fetch_timeout not implemented. } void expectDisableInitFetchTimeoutTimer() override { diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index b266dda51b0c..af27a0ba6192 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -89,7 +89,7 @@ class GrpcMuxImplTestBase : public testing::Test { } NiceMock dispatcher_; - Runtime::MockRandomGenerator random_; + NiceMock random_; Grpc::MockAsyncClient* async_client_; Grpc::MockAsyncStream async_stream_; std::unique_ptr grpc_mux_; @@ -213,13 +213,15 @@ TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { invalid_response->mutable_resources()->Add()->set_type_url("bar"); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)) .WillOnce(Invoke([](Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) { - EXPECT_TRUE(IsSubstring("", "", "bar does not match foo type URL in DiscoveryResponse", - e->what())); + EXPECT_TRUE(IsSubstring( + "", "", "bar does not match the message-wide type URL foo in DiscoveryResponse", + e->what())); })); - expectSendMessage("foo", {"x", "y"}, "", false, "", Grpc::Status::GrpcStatus::Internal, - fmt::format("bar does not match foo type URL in DiscoveryResponse {}", - invalid_response->DebugString())); + expectSendMessage( + "foo", {"x", "y"}, "", false, "", Grpc::Status::GrpcStatus::Internal, + fmt::format("bar does not match the message-wide type URL foo in DiscoveryResponse {}", + invalid_response->DebugString())); grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(invalid_response)); } expectSendMessage("foo", {}, ""); diff --git a/test/common/config/grpc_stream_test.cc b/test/common/config/grpc_stream_test.cc index 783f188eefd6..e54f901ddee6 100644 --- a/test/common/config/grpc_stream_test.cc +++ b/test/common/config/grpc_stream_test.cc @@ -43,7 +43,7 @@ class GrpcStreamTest : public testing::Test { // Tests that establishNewStream() establishes it, a second call does nothing, and a third call // after the stream was disconnected re-establishes it. -TEST_F(GrpcStreamTest, EstablishNewStream) { +TEST_F(GrpcStreamTest, EstablishStream) { EXPECT_FALSE(grpc_stream_.grpcStreamAvailable()); // Successful establishment { @@ -52,7 +52,7 @@ TEST_F(GrpcStreamTest, EstablishNewStream) { grpc_stream_.establishNewStream(); EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); } - // Idempotency: do nothing (other than logging a warning) if already connected + // Idempotent { EXPECT_CALL(*async_client_, startRaw(_, _, _)).Times(0); EXPECT_CALL(callbacks_, onStreamEstablished()).Times(0); diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index e79995b28f39..56c86c18dca6 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -25,7 +25,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); // Ensure this doesn't cause an issue by sending a request, since we don't // have a gRPC stream. - subscription_->updateResources({"cluster2"}); + subscription_->updateResourceInterest({"cluster2"}); // Retry and succeed. EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); @@ -46,8 +46,8 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { .Times(0); EXPECT_CALL(*timer_, enableTimer(_, _)); EXPECT_CALL(random_, random()); - subscription_->grpcMux().grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, - ""); + subscription_->grpcMux()->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, + ""); EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); verifyControlPlaneStats(0); @@ -65,14 +65,14 @@ TEST_F(GrpcSubscriptionImplTest, RepeatedNonce) { startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); // First with the initial, empty version update to "0". - updateResources({"cluster2"}); + updateResourceInterest({"cluster2"}); EXPECT_TRUE(statsAre(2, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster2"}, "0", false); EXPECT_TRUE(statsAre(3, 0, 1, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster2"}, "0", true); EXPECT_TRUE(statsAre(4, 1, 1, 0, 0, 7148434200721666028)); // Now with version "0" update to "1". - updateResources({"cluster3"}); + updateResourceInterest({"cluster3"}); EXPECT_TRUE(statsAre(5, 1, 1, 0, 0, 7148434200721666028)); deliverConfigUpdate({"cluster3"}, "1", false); EXPECT_TRUE(statsAre(6, 1, 2, 0, 0, 7148434200721666028)); diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 9e68838ee646..fb985f68ed7d 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -114,11 +114,11 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { expectSendMessage(last_cluster_names_, version_, false, Grpc::Status::GrpcStatus::Internal, "bad config"); } - subscription_->grpcMux().onDiscoveryResponse(std::move(response)); + subscription_->grpcMux()->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } - void updateResources(const std::set& cluster_names) override { + void updateResourceInterest(const std::set& cluster_names) override { // The "watch" mechanism means that updates that lose interest in a resource // will first generate a request for [still watched resources, i.e. without newly unwatched // ones] before generating the request for all of cluster_names. @@ -132,17 +132,22 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { } expectSendMessage(both, version_); expectSendMessage(cluster_names, version_); - subscription_->updateResources(cluster_names); + subscription_->updateResourceInterest(cluster_names); last_cluster_names_ = cluster_names; } void expectConfigUpdateFailed() override { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)) + .WillOnce([this](ConfigUpdateFailureReason reason, const EnvoyException*) { + if (reason == ConfigUpdateFailureReason::FetchTimedout) { + stats_.init_fetch_timeout_.inc(); + } + }); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { init_timeout_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout), _)); + EXPECT_CALL(*init_timeout_timer_, enableTimer(timeout, _)); } void expectDisableInitFetchTimeoutTimer() override { diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index 1d41ee38019c..1f00c95cd87d 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -106,10 +106,10 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { subscription_->start(cluster_names); } - void updateResources(const std::set& cluster_names) override { + void updateResourceInterest(const std::set& cluster_names) override { cluster_names_ = cluster_names; expectSendMessage(cluster_names, version_); - subscription_->updateResources(cluster_names); + subscription_->updateResourceInterest(cluster_names); timer_cb_(); } diff --git a/test/common/config/new_grpc_mux_impl_test.cc b/test/common/config/new_grpc_mux_impl_test.cc new file mode 100644 index 000000000000..98150c9b7915 --- /dev/null +++ b/test/common/config/new_grpc_mux_impl_test.cc @@ -0,0 +1,111 @@ +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/api/v2/eds.pb.h" + +#include "common/common/empty_string.h" +#include "common/config/new_grpc_mux_impl.h" +#include "common/config/protobuf_link_hacks.h" +#include "common/config/resources.h" +#include "common/config/utility.h" +#include "common/protobuf/protobuf.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/mocks/common.h" +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/test_common/logging.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_time.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Invoke; +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Config { +namespace { + +// We test some mux specific stuff below, other unit test coverage for singleton use of +// NewGrpcMuxImpl is provided in [grpc_]subscription_impl_test.cc. +class NewGrpcMuxImplTestBase : public testing::Test { +public: + NewGrpcMuxImplTestBase() + : async_client_(new Grpc::MockAsyncClient()), + control_plane_connected_state_( + stats_.gauge("control_plane.connected_state", Stats::Gauge::ImportMode::NeverImport)) {} + + void setup() { + grpc_mux_ = std::make_unique( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + random_, stats_, rate_limit_settings_, local_info_); + } + + NiceMock dispatcher_; + NiceMock random_; + Grpc::MockAsyncClient* async_client_; + NiceMock async_stream_; + std::unique_ptr grpc_mux_; + NiceMock> callbacks_; + NiceMock local_info_; + Stats::IsolatedStoreImpl stats_; + Envoy::Config::RateLimitSettings rate_limit_settings_; + Stats::Gauge& control_plane_connected_state_; +}; + +class NewGrpcMuxImplTest : public NewGrpcMuxImplTestBase { +public: + Event::SimulatedTimeSystem time_system_; +}; + +// Test that we simply ignore a message for an unknown type_url, with no ill effects. +TEST_F(NewGrpcMuxImplTest, DiscoveryResponseNonexistentSub) { + setup(); + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + grpc_mux_->addOrUpdateWatch(type_url, nullptr, {}, callbacks_, std::chrono::milliseconds(0)); + + EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); + grpc_mux_->start(); + + { + auto unexpected_response = std::make_unique(); + unexpected_response->set_type_url(type_url); + unexpected_response->set_system_version_info("0"); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "0")).Times(0); + grpc_mux_->onDiscoveryResponse(std::move(unexpected_response)); + } + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_system_version_info("1"); + envoy::api::v2::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->mutable_resource()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "1")) + .WillOnce( + Invoke([&load_assignment]( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField&, const std::string&) { + EXPECT_EQ(1, added_resources.size()); + envoy::api::v2::ClusterLoadAssignment expected_assignment; + added_resources[0].resource().UnpackTo(&expected_assignment); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + grpc_mux_->onDiscoveryResponse(std::move(response)); + } +} + +} // namespace +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/subscription_factory_impl_test.cc b/test/common/config/subscription_factory_impl_test.cc index 403ad4d5f91f..2426c10d8395 100644 --- a/test/common/config/subscription_factory_impl_test.cc +++ b/test/common/config/subscription_factory_impl_test.cc @@ -38,7 +38,7 @@ class SubscriptionFactoryTest : public testing::Test { return SubscriptionFactoryImpl(local_info_, dispatcher_, cm_, random_, validation_visitor_, *api_) .subscriptionFromConfigSource(config, Config::TypeUrl::get().ClusterLoadAssignment, - stats_store_, callbacks_); + stats_store_, callbacks_, false); } Upstream::MockClusterManager cm_; diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index c45f19bd00e8..45646de9b531 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -39,12 +39,14 @@ class SubscriptionImplTest : public testing::TestWithParam { } } + void TearDown() override { test_harness_->doSubscriptionTearDown(); } + void startSubscription(const std::set& cluster_names) { test_harness_->startSubscription(cluster_names); } - void updateResources(const std::set& cluster_names) { - test_harness_->updateResources(cluster_names); + void updateResourceInterest(const std::set& cluster_names) { + test_harness_->updateResourceInterest(cluster_names); } void expectSendMessage(const std::set& cluster_names, const std::string& version, @@ -90,72 +92,75 @@ INSTANTIATE_TEST_SUITE_P(SubscriptionImplTest, SubscriptionImplInitFetchTimeoutT // Validate basic request-response succeeds. TEST_P(SubscriptionImplTest, InitialRequestResponse) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(2, 1, 0, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(2, 1, 0, 0, 0, 7148434200721666028)); } // Validate that multiple streamed updates succeed. TEST_P(SubscriptionImplTest, ResponseStream) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(2, 1, 0, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(2, 1, 0, 0, 0, 7148434200721666028)); deliverConfigUpdate({"cluster0", "cluster1"}, "1", true); - statsAre(3, 2, 0, 0, 0, 13237225503670494420U); + EXPECT_TRUE(statsAre(3, 2, 0, 0, 0, 13237225503670494420U)); } // Validate that the client can reject a config. TEST_P(SubscriptionImplTest, RejectConfig) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - statsAre(2, 0, 1, 0, 0, 0); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0)); } // Validate that the client can reject a config and accept the same config later. TEST_P(SubscriptionImplTest, RejectAcceptConfig) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - statsAre(2, 0, 1, 0, 0, 0); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(3, 1, 1, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(3, 1, 1, 0, 0, 7148434200721666028)); } // Validate that the client can reject a config and accept another config later. TEST_P(SubscriptionImplTest, RejectAcceptNextConfig) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - statsAre(2, 0, 1, 0, 0, 0); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "1", true); - statsAre(3, 1, 1, 0, 0, 13237225503670494420U); + EXPECT_TRUE(statsAre(3, 1, 1, 0, 0, 13237225503670494420U)); } // Validate that stream updates send a message with the updated resources. TEST_P(SubscriptionImplTest, UpdateResources) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(2, 1, 0, 0, 0, 7148434200721666028); - updateResources({"cluster2"}); - statsAre(3, 1, 0, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(2, 1, 0, 0, 0, 7148434200721666028)); + updateResourceInterest({"cluster2"}); + EXPECT_TRUE(statsAre(3, 1, 0, 0, 0, 7148434200721666028)); } // Validate that initial fetch timer is created and calls callback on timeout TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { + if (GetParam() == SubscriptionType::Filesystem) { + return; // initial_fetch_timeout not implemented for filesystem. + } InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); if (GetParam() == SubscriptionType::Http) { expectDisableInitFetchTimeoutTimer(); } expectConfigUpdateFailed(); callInitFetchTimeoutCb(); - statsAre(1, 0, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 1, 0)); } // Validate that initial fetch timer is disabled on config update @@ -163,7 +168,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnSuccess) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); expectDisableInitFetchTimeoutTimer(); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); } @@ -173,7 +178,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnFail) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); expectDisableInitFetchTimeoutTimer(); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); } diff --git a/test/common/config/subscription_test_harness.h b/test/common/config/subscription_test_harness.h index 6a094116324d..0f3b6c44da35 100644 --- a/test/common/config/subscription_test_harness.h +++ b/test/common/config/subscription_test_harness.h @@ -30,7 +30,7 @@ class SubscriptionTestHarness { * Update cluster names to be delivered via EDS. * @param cluster_names cluster names. */ - virtual void updateResources(const std::set& cluster_names) PURE; + virtual void updateResourceInterest(const std::set& cluster_names) PURE; /** * Expect that an update request is sent by the Subscription implementation. @@ -94,6 +94,8 @@ class SubscriptionTestHarness { virtual void callInitFetchTimeoutCb() PURE; + virtual void doSubscriptionTearDown() {} + Stats::IsolatedStoreImpl stats_store_; SubscriptionStats stats_; }; diff --git a/test/common/grpc/grpc_client_integration.h b/test/common/grpc/grpc_client_integration.h index bdfc0c6ae1ba..86017cf821d1 100644 --- a/test/common/grpc/grpc_client_integration.h +++ b/test/common/grpc/grpc_client_integration.h @@ -53,34 +53,23 @@ class GrpcClientIntegrationParamTest ClientType clientType() const override { return std::get<1>(GetParam()); } }; -class DeltaSotwGrpcClientIntegrationParamTest +class DeltaSotwIntegrationParamTest : public BaseGrpcClientIntegrationParamTest, - public testing::TestWithParam> { + public testing::TestWithParam< + std::tuple> { public: - static std::string protocolTestParamsToString( - const ::testing::TestParamInfo>& - p) { - return fmt::format("{}_{}", + ~DeltaSotwIntegrationParamTest() override = default; + static std::string + protocolTestParamsToString(const ::testing::TestParamInfo< + std::tuple>& p) { + return fmt::format("{}_{}_{}", std::get<0>(p.param) == Network::Address::IpVersion::v4 ? "IPv4" : "IPv6", std::get<1>(p.param) == ClientType::GoogleGrpc ? "GoogleGrpc" : "EnvoyGrpc", - std::get<2>(p.param) ? "Delta" : "StateOfTheWorld"); + std::get<2>(p.param) == SotwOrDelta::Delta ? "Delta" : "StateOfTheWorld"); } Network::Address::IpVersion ipVersion() const override { return std::get<0>(GetParam()); } ClientType clientType() const override { return std::get<1>(GetParam()); } - bool isDelta() { return std::get<2>(GetParam()); } -}; - -class DeltaSotwIntegrationParamTest - : public testing::TestWithParam> { -public: - static std::string protocolTestParamsToString( - const ::testing::TestParamInfo>& p) { - return fmt::format("{}_{}_{}", - std::get<0>(p.param) == Network::Address::IpVersion::v4 ? "IPv4" : "IPv6", - std::get<1>(p.param) == SotwOrDelta::Delta ? "Delta" : "StateOfTheWorld"); - } - Network::Address::IpVersion ipVersion() const { return std::get<0>(GetParam()); } - SotwOrDelta sotwOrDelta() const { return std::get<1>(GetParam()); } + SotwOrDelta sotwOrDelta() const { return std::get<2>(GetParam()); } }; // Skip tests based on gRPC client type. @@ -102,19 +91,16 @@ class DeltaSotwIntegrationParamTest #define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc, Grpc::ClientType::GoogleGrpc), \ - testing::Bool()) + testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) #else #define GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc)) #define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ - testing::Values(Grpc::ClientType::EnvoyGrpc), testing::Bool()) -#endif // ENVOY_GOOGLE_GRPC - -#define DELTA_INTEGRATION_PARAMS \ - testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ + testing::Values(Grpc::ClientType::EnvoyGrpc), \ testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) +#endif // ENVOY_GOOGLE_GRPC } // namespace Grpc } // namespace Envoy diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index 11424515eb32..9be73fbefe8c 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -92,9 +92,9 @@ class RdsImplTest : public RdsTestBase { )EOF"; EXPECT_CALL(factory_context_.init_manager_, add(_)); - rds_ = - RouteConfigProviderUtil::create(parseHttpConnectionManagerFromJson(config_json), - factory_context_, "foo.", *route_config_provider_manager_); + rds_ = RouteConfigProviderUtil::create(parseHttpConnectionManagerFromJson(config_json), + factory_context_, "foo.", + *route_config_provider_manager_, false); rds_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; EXPECT_CALL(*factory_context_.cluster_manager_.subscription_factory_.subscription_, start(_)); factory_context_.init_manager_.initialize(init_watcher_); @@ -126,7 +126,7 @@ TEST_F(RdsImplTest, RdsAndStatic) { EXPECT_THROW(RouteConfigProviderUtil::create(parseHttpConnectionManagerFromJson(config_json), factory_context_, "foo.", - *route_config_provider_manager_), + *route_config_provider_manager_, false), EnvoyException); } @@ -264,7 +264,7 @@ class RouteConfigProviderManagerImplTest : public RdsTestBase { rds_.set_route_config_name("foo_route_config"); rds_.mutable_config_source()->set_path("foo_path"); provider_ = route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, factory_context_, "foo_prefix.", factory_context_.initManager()); + rds_, factory_context_, "foo_prefix.", factory_context_.initManager(), false); rds_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; } @@ -416,7 +416,7 @@ name: foo_route_config "1"); RouteConfigProviderPtr provider2 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, factory_context_, "foo_prefix", factory_context_.initManager()); + rds_, factory_context_, "foo_prefix", factory_context_.initManager(), false); // provider2 should have route config immediately after create EXPECT_TRUE(provider2->configInfo().has_value()); @@ -430,7 +430,7 @@ name: foo_route_config rds2.set_route_config_name("foo_route_config"); rds2.mutable_config_source()->set_path("bar_path"); RouteConfigProviderPtr provider3 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds2, factory_context_, "foo_prefix", factory_context_.initManager()); + rds2, factory_context_, "foo_prefix", factory_context_.initManager(), false); EXPECT_NE(provider3, provider_); factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(route_configs, "provider3"); diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 3967703c4125..d8a3fcd5ff23 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -6,6 +6,7 @@ #include "envoy/init/manager.h" #include "envoy/stats/scope.h" +#include "common/config/grpc_mux_impl.h" #include "common/router/scoped_rds.h" #include "test/mocks/config/mocks.h" @@ -94,25 +95,27 @@ class ScopedRoutesTestBase : public testing::Test { class ScopedRdsTest : public ScopedRoutesTestBase { protected: void setup() { - InSequence s; + ON_CALL(factory_context_.cluster_manager_, adsMux()) + .WillByDefault(Return(std::make_shared<::Envoy::Config::NullGrpcMuxImpl>())); + InSequence s; // Since factory_context_.cluster_manager_.subscription_factory_.callbacks_ is taken by the SRDS // subscription. We need to return a different MockSubscription here for each RDS subscription. // To build the map from RDS route_config_name to the RDS subscription, we need to get the // route_config_name by mocking start() on the Config::Subscription. EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, - subscriptionFromConfigSource(_, _, _, _)) + subscriptionFromConfigSource(_, _, _, _, _)) .Times(AnyNumber()); EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, subscriptionFromConfigSource( _, Eq(Grpc::Common::typeUrl( envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name())), - _, _)) + _, _, _)) .Times(AnyNumber()) .WillRepeatedly(Invoke([this](const envoy::api::v2::core::ConfigSource&, absl::string_view, Stats::Scope&, - Envoy::Config::SubscriptionCallbacks& callbacks) { + Envoy::Config::SubscriptionCallbacks& callbacks, bool) { auto ret = std::make_unique>(); rds_subscription_by_config_subscription_[ret.get()] = &callbacks; EXPECT_CALL(*ret, start(_)) diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index ef5428e19ea9..182b92d1ef5d 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -808,10 +808,10 @@ class RtdsLoaderImplTest : public LoaderImplTest { EXPECT_CALL(init_manager_, add(_)).WillRepeatedly(Invoke([this](const Init::Target& target) { init_target_handles_.emplace_back(target.createHandle("test")); })); - ON_CALL(cm_.subscription_factory_, subscriptionFromConfigSource(_, _, _, _)) + ON_CALL(cm_.subscription_factory_, subscriptionFromConfigSource(_, _, _, _, _)) .WillByDefault(testing::Invoke( [this](const envoy::api::v2::core::ConfigSource&, absl::string_view, Stats::Scope&, - Config::SubscriptionCallbacks& callbacks) -> Config::SubscriptionPtr { + Config::SubscriptionCallbacks& callbacks, bool) -> Config::SubscriptionPtr { auto ret = std::make_unique>(); rtds_subscriptions_.push_back(ret.get()); rtds_callbacks_.push_back(&callbacks); diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index fe6b5e39ab60..6c0138cbaf04 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -32,9 +32,9 @@ MATCHER_P(WithName, expectedName, "") { return arg.name() == expectedName; } class CdsApiImplTest : public testing::Test { protected: - void setup() { + void setup(bool is_delta = false) { envoy::api::v2::core::ConfigSource cds_config; - cds_ = CdsApiImpl::create(cds_config, cm_, store_, validation_visitor_); + cds_ = CdsApiImpl::create(cds_config, is_delta, cm_, store_, validation_visitor_); cds_->setInitializedCb([this]() -> void { initialized_.ready(); }); EXPECT_CALL(*cm_.subscription_factory_.subscription_, start(_)); @@ -180,7 +180,7 @@ TEST_F(CdsApiImplTest, ConfigUpdateWith2ValidClusters) { TEST_F(CdsApiImplTest, DeltaConfigUpdate) { { InSequence s; - setup(); + setup(true); } EXPECT_CALL(initialized_, ready()); diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 3aed95db6298..4ea0c9080cd1 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -101,7 +101,7 @@ class TestClusterManagerFactory : public ClusterManagerFactory { return std::make_pair(result.first, ThreadAwareLoadBalancerPtr(result.second)); } - CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource&, ClusterManager&) override { + CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource&, bool, ClusterManager&) override { return CdsApiPtr{createCds_()}; } @@ -2972,12 +2972,7 @@ TEST_F(ClusterManagerInitHelperTest, InitSecondaryWithoutEdsPaused) { ON_CALL(cluster1, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); init_helper_.addCluster(cluster1); - const auto& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - EXPECT_CALL(cm_.ads_mux_, paused(Eq(ByRef(type_url)))).WillRepeatedly(Return(false)); - EXPECT_CALL(cm_.ads_mux_, pause(Eq(ByRef(type_url)))); EXPECT_CALL(cluster1, initialize(_)); - EXPECT_CALL(cm_.ads_mux_, resume(Eq(ByRef(type_url)))); - init_helper_.onStaticLoadComplete(); EXPECT_CALL(*this, onClusterInit(Ref(cluster1))); @@ -2999,12 +2994,7 @@ TEST_F(ClusterManagerInitHelperTest, InitSecondaryWithEdsPaused) { ON_CALL(cluster1, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); init_helper_.addCluster(cluster1); - const auto& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - EXPECT_CALL(cm_.ads_mux_, paused(Eq(ByRef(type_url)))).WillRepeatedly(Return(true)); - EXPECT_CALL(cm_.ads_mux_, pause(Eq(ByRef(type_url)))).Times(0); EXPECT_CALL(cluster1, initialize(_)); - EXPECT_CALL(cm_.ads_mux_, resume(Eq(ByRef(type_url)))).Times(0); - init_helper_.onStaticLoadComplete(); EXPECT_CALL(*this, onClusterInit(Ref(cluster1))); diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index 9247c9d84eb8..1e520b815163 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -75,8 +75,8 @@ class EdsTest : public testing::Test { Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( admin_, ssl_context_manager_, *scope, cm_, local_info_, dispatcher_, random_, stats_, singleton_manager_, tls_, validation_visitor_, *api_); - cluster_.reset( - new EdsClusterImpl(eds_cluster_, runtime_, factory_context, std::move(scope), false)); + cluster_.reset(new EdsClusterImpl(eds_cluster_, runtime_, factory_context, std::move(scope), + false, false)); EXPECT_EQ(Cluster::InitializePhase::Secondary, cluster_->initializePhase()); eds_callbacks_ = cm_.subscription_factory_.callbacks_; } diff --git a/test/config/utility.cc b/test/config/utility.cc index c230b4c772c3..e93bb5c333dd 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -155,6 +155,8 @@ name: envoy.squash nanos: 0 )EOF"; +// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification +// work restores it here. // TODO(#6327) cleaner approach to testing with static config. std::string ConfigHelper::discoveredClustersBootstrap(const std::string& api_type) { return fmt::format( @@ -172,7 +174,7 @@ std::string ConfigHelper::discoveredClustersBootstrap(const std::string& api_typ grpc_services: envoy_grpc: cluster_name: my_cds_cluster - set_node_on_first_message_only: true + set_node_on_first_message_only: false static_resources: clusters: - name: my_cds_cluster @@ -214,6 +216,39 @@ std::string ConfigHelper::discoveredClustersBootstrap(const std::string& api_typ api_type); } +// TODO(#6327) cleaner approach to testing with static config. +std::string ConfigHelper::adsBootstrap(const std::string& api_type) { + return fmt::format( + R"EOF( +dynamic_resources: + lds_config: + ads: {{}} + cds_config: + ads: {{}} + ads_config: + api_type: {} +static_resources: + clusters: + name: dummy_cluster + connect_timeout: + seconds: 5 + type: STATIC + hosts: + socket_address: + address: 127.0.0.1 + port_value: 0 + lb_policy: ROUND_ROBIN + http2_protocol_options: {{}} +admin: + access_log_path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 0 +)EOF", + api_type); +} + envoy::api::v2::Cluster ConfigHelper::buildCluster(const std::string& name, int port, const std::string& ip_version) { return TestUtility::parseYaml(fmt::format(R"EOF( diff --git a/test/config/utility.h b/test/config/utility.h index 574160651aa3..00cb9aece227 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -91,6 +91,7 @@ class ConfigHelper { // Configuration for L7 proxying, with clusters cluster_1 and cluster_2 meant to be added via CDS. // api_type should be REST, GRPC, or DELTA_GRPC. static std::string discoveredClustersBootstrap(const std::string& api_type); + static std::string adsBootstrap(const std::string& api_type); // Builds a standard Cluster config fragment, with a single endpoint (at loopback:port). static envoy::api::v2::Cluster buildCluster(const std::string& name, int port, const std::string& ip_version); diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index 4e2590e4b41d..6138f963e13a 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -19,10 +19,13 @@ using testing::AssertionResult; namespace Envoy { AdsIntegrationTest::AdsIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), AdsIntegrationConfig()) { + : HttpIntegrationTest( + Http::CodecClient::Type::HTTP2, ipVersion(), + AdsIntegrationConfig(sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { use_lds_ = false; create_xds_upstream_ = true; tls_xds_upstream_ = true; + sotw_or_delta_ = sotwOrDelta(); } void AdsIntegrationTest::TearDown() { diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h index 51a229e9dfe3..31956bd6bb12 100644 --- a/test/integration/ads_integration.h +++ b/test/integration/ads_integration.h @@ -12,37 +12,43 @@ #include "test/common/grpc/grpc_client_integration.h" #include "test/integration/http_integration.h" +// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification +// work restores it here. namespace Envoy { -static std::string AdsIntegrationConfig() { +static std::string AdsIntegrationConfig(const std::string& api_type) { // Note: do not use CONSTRUCT_ON_FIRST_USE here! - return R"EOF( + return fmt::format(R"EOF( dynamic_resources: - lds_config: {ads: {}} - cds_config: {ads: {}} + lds_config: + ads: {{}} + cds_config: + ads: {{}} ads_config: - api_type: GRPC - set_node_on_first_message_only: true + api_type: {} + set_node_on_first_message_only: false static_resources: clusters: name: dummy_cluster - connect_timeout: { seconds: 5 } + connect_timeout: + seconds: 5 type: STATIC hosts: socket_address: address: 127.0.0.1 port_value: 0 lb_policy: ROUND_ROBIN - http2_protocol_options: {} + http2_protocol_options: {{}} admin: access_log_path: /dev/null address: socket_address: address: 127.0.0.1 port_value: 0 -)EOF"; +)EOF", + api_type); } -class AdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public HttpIntegrationTest { +class AdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsIntegrationTest(); diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index e2cb9250c0e6..442b5dd8293b 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -26,7 +26,8 @@ using testing::AssertionResult; namespace Envoy { -INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsIntegrationTest, GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // Validate basic config delivery and upgrade. TEST_P(AdsIntegrationTest, Basic) { @@ -54,9 +55,8 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Cluster, "", {}, {}, {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().ClusterLoadAssignment, - Config::TypeUrl::get().Cluster))); + Config::TypeUrl::get().Cluster, "", {}, {}, {}, true, Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().Cluster))); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); @@ -68,11 +68,11 @@ TEST_P(AdsIntegrationTest, Failure) { {buildCluster("cluster_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); - EXPECT_TRUE( - compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", {"cluster_0"}, {}, - {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().Cluster, - Config::TypeUrl::get().ClusterLoadAssignment))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", + {"cluster_0"}, {}, {}, true, + Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", + Config::TypeUrl::get().ClusterLoadAssignment))); sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("cluster_0")}, {buildClusterLoadAssignment("cluster_0")}, {}, "1"); @@ -84,9 +84,8 @@ TEST_P(AdsIntegrationTest, Failure) { {buildRouteConfig("listener_0", "route_config_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Listener, "", {}, {}, {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().RouteConfiguration, - Config::TypeUrl::get().Listener))); + Config::TypeUrl::get().Listener, "", {}, {}, {}, true, Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().Listener))); sendDiscoveryResponse( Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, {buildListener("listener_0", "route_config_0")}, {}, "1"); @@ -98,11 +97,11 @@ TEST_P(AdsIntegrationTest, Failure) { {buildListener("route_config_0", "cluster_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); - EXPECT_TRUE( - compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", {"route_config_0"}, {}, - {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().Listener, - Config::TypeUrl::get().RouteConfiguration))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", + {"route_config_0"}, {}, {}, true, + Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", + Config::TypeUrl::get().RouteConfiguration))); sendDiscoveryResponse( Config::TypeUrl::get().RouteConfiguration, {buildRouteConfig("route_config_0", "cluster_0")}, {buildRouteConfig("route_config_0", "cluster_0")}, {}, "1"); @@ -111,6 +110,7 @@ TEST_P(AdsIntegrationTest, Failure) { {"route_config_0"}, {}, {})); test_server_->waitForCounterGe("listener_manager.listener_create_success", 1); + makeSingleRequest(); } @@ -209,7 +209,7 @@ TEST_P(AdsIntegrationTest, RedisClusterRemoval) { {buildRedisCluster("redis_cluster")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", - {"redis_cluster"}, {}, {})); + {"redis_cluster"}, {"redis_cluster"}, {})); sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("redis_cluster")}, {buildClusterLoadAssignment("redis_cluster")}, {}, "1"); @@ -261,7 +261,7 @@ TEST_P(AdsIntegrationTest, DuplicateWarmingClusters) { {buildListener("listener_0", "route_config_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", - {"cluster_0"}, {"cluster_0"}, {})); + {"cluster_0"}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", {"route_config_0"}, {"route_config_0"}, {})); sendDiscoveryResponse( @@ -321,30 +321,32 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { makeSingleRequest(); EXPECT_FALSE( - test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); + test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); // Send the first warming cluster. sendDiscoveryResponse( Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_1")}, {buildCluster("warming_cluster_1")}, {"cluster_0"}, "2"); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); EXPECT_TRUE( - test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); + test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", {"warming_cluster_1"}, {"warming_cluster_1"}, {"cluster_0"})); // Send the second warming cluster. - sendDiscoveryResponse( - Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_2")}, - {buildCluster("warming_cluster_2")}, {"warming_cluster_1"}, "3"); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildCluster("warming_cluster_2")}, + {buildCluster("warming_cluster_2")}, {}, "3"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2); // We would've got a Cluster discovery request with version 2 here, had the CDS not been paused. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", {"warming_cluster_2", "warming_cluster_1"}, {"warming_cluster_2"}, {})); EXPECT_TRUE( - test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); + test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); // Finish warming the clusters. sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, @@ -357,9 +359,15 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { // Validate that clusters are warmed. test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); EXPECT_FALSE( - test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); + test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); // CDS is resumed and EDS response was acknowledged. + if (sotw_or_delta_ == Grpc::SotwOrDelta::Delta) { + // Envoy will ACK both Cluster messages. Since they arrived while CDS was paused, they aren't + // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 + // ACK goes out, they're both acknowledging version 3. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); + } EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "2", {"warming_cluster_2", "warming_cluster_1"}, {}, {})); @@ -412,9 +420,9 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { {"warming_cluster_1"}, {"warming_cluster_1"}, {"cluster_0"})); // Send the second warming cluster. - sendDiscoveryResponse( - Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_2")}, - {buildCluster("warming_cluster_2")}, {"warming_cluster_1"}, "3"); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildCluster("warming_cluster_2")}, + {buildCluster("warming_cluster_2")}, {}, "3"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", @@ -425,7 +433,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("warming_cluster_1")}, - {buildClusterLoadAssignment("warming_cluster_1")}, {"cluster_0"}, "2"); + {buildClusterLoadAssignment("warming_cluster_1")}, {}, "2"); // Envoy will not finish warming of the second cluster because of the missing load assignments // i,e. no named EDS response. @@ -446,7 +454,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("warming_cluster_2")}, - {buildClusterLoadAssignment("warming_cluster_2")}, {"warming_cluster_1"}, "3"); + {buildClusterLoadAssignment("warming_cluster_2")}, {}, "3"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); } @@ -580,13 +588,16 @@ TEST_P(AdsIntegrationTest, RdsAfterLdsInvalidated) { test_server_->waitForCounterGe("listener_manager.listener_create_success", 2); } -class AdsFailIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, +class AdsFailIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsFailIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), AdsIntegrationConfig()) { + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + AdsIntegrationConfig( + sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { create_xds_upstream_ = true; use_lds_ = false; + sotw_or_delta_ = sotwOrDelta(); } void TearDown() override { @@ -609,8 +620,8 @@ class AdsFailIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, } }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsFailIntegrationTest, - GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsFailIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // Validate that we don't crash on failed ADS stream. TEST_P(AdsFailIntegrationTest, ConnectDisconnect) { @@ -621,13 +632,16 @@ TEST_P(AdsFailIntegrationTest, ConnectDisconnect) { xds_stream_->finishGrpcStream(Grpc::Status::Internal); } -class AdsConfigIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, +class AdsConfigIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsConfigIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), AdsIntegrationConfig()) { + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + AdsIntegrationConfig( + sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { create_xds_upstream_ = true; use_lds_ = false; + sotw_or_delta_ = sotwOrDelta(); } void TearDown() override { @@ -658,8 +672,8 @@ class AdsConfigIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, } }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsConfigIntegrationTest, - GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsConfigIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // This is s regression validating that we don't crash on EDS static Cluster that uses ADS. TEST_P(AdsConfigIntegrationTest, EdsClusterWithAdsConfigSource) { @@ -753,7 +767,8 @@ TEST_P(AdsIntegrationTest, ListenerDrainBeforeServerStart) { // Remove listener. EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); - sendDiscoveryResponse(Config::TypeUrl::get().Listener, {}, {}, {}, "1"); + sendDiscoveryResponse(Config::TypeUrl::get().Listener, {}, {}, + {"listener_0"}, "2"); test_server_->waitForGaugeEq("listener_manager.total_listeners_active", 0); } diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index 1ea5f5fe67e8..d3013870ea94 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -122,7 +122,8 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht bool test_skipped_{true}; }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, CdsIntegrationTest, DELTA_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, CdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // 1) Envoy starts up with no static clusters (other than the CDS-over-gRPC server). // 2) Envoy is told of a cluster via CDS. diff --git a/test/integration/integration.cc b/test/integration/integration.cc index 595c78f1e24b..cf4baa464d10 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -554,21 +554,21 @@ AssertionResult BaseIntegrationTest::compareDiscoveryRequest( const std::vector& expected_resource_names, const std::vector& expected_resource_names_added, const std::vector& expected_resource_names_removed, bool expect_node, - const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { + const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw) { return compareSotwDiscoveryRequest(expected_type_url, expected_version, expected_resource_names, - expect_node, expected_error_code, expected_error_message); + expect_node, expected_error_code, expected_error_substring); } else { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_names_added, expected_resource_names_removed, expected_error_code, - expected_error_message); + expected_error_substring); } } AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, const std::vector& expected_resource_names, bool expect_node, - const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { + const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { envoy::api::v2::DiscoveryRequest discovery_request; VERIFY_ASSERTION(xds_stream_->waitForGrpcMessage(*dispatcher_, discovery_request)); @@ -590,7 +590,7 @@ AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( expected_error_code); } EXPECT_TRUE( - IsSubstring("", "", expected_error_message, discovery_request.error_detail().message())); + IsSubstring("", "", expected_error_substring, discovery_request.error_detail().message())); const std::vector resource_names(discovery_request.resource_names().cbegin(), discovery_request.resource_names().cend()); if (expected_resource_names != resource_names) { @@ -608,47 +608,69 @@ AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( return AssertionSuccess(); } +AssertionResult compareSets(const std::set& set1, const std::set& set2, + absl::string_view name) { + if (set1 == set2) { + return AssertionSuccess(); + } + auto failure = AssertionFailure() << name << " field not as expected.\nExpected: {"; + for (const auto& x : set1) { + failure << x << ", "; + } + failure << "}\nActual: {"; + for (const auto& x : set2) { + failure << x << ", "; + } + return failure << "}"; +} + AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& xds_stream, - const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { + const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { envoy::api::v2::DeltaDiscoveryRequest request; VERIFY_ASSERTION(xds_stream->waitForGrpcMessage(*dispatcher_, request)); - EXPECT_TRUE(request.has_node()); - EXPECT_FALSE(request.node().id().empty()); - EXPECT_FALSE(request.node().cluster().empty()); - - if (expected_type_url != request.type_url()) { - return AssertionFailure() << fmt::format("type_url {} does not match expected {}", - request.type_url(), expected_type_url); + // Verify all we care about node. + if (!request.has_node() || request.node().id().empty() || request.node().cluster().empty()) { + return AssertionFailure() << "Weird node field"; } - if (!(expected_error_code == request.error_detail().code())) { - return AssertionFailure() << fmt::format("error_code {} does not match expected {}", - request.error_detail().code(), expected_error_code); + if (request.type_url() != expected_type_url) { + return AssertionFailure() << fmt::format("type_url {} does not match expected {}.", + request.type_url(), expected_type_url); } - EXPECT_TRUE(IsSubstring("", "", expected_error_message, request.error_detail().message())); - - const std::vector resource_subscriptions(request.resource_names_subscribe().cbegin(), - request.resource_names_subscribe().cend()); - if (expected_resource_subscriptions != resource_subscriptions) { - return AssertionFailure() << fmt::format( - "newly subscribed resources {} do not match expected {} in {}", - fmt::join(resource_subscriptions.begin(), resource_subscriptions.end(), ","), - fmt::join(expected_resource_subscriptions.begin(), - expected_resource_subscriptions.end(), ","), - request.DebugString()); - } - const std::vector resource_unsubscriptions( - request.resource_names_unsubscribe().cbegin(), request.resource_names_unsubscribe().cend()); - if (expected_resource_unsubscriptions != resource_unsubscriptions) { + // Sort to ignore ordering. + std::set expected_sub{expected_resource_subscriptions.begin(), + expected_resource_subscriptions.end()}; + std::set expected_unsub{expected_resource_unsubscriptions.begin(), + expected_resource_unsubscriptions.end()}; + std::set actual_sub{request.resource_names_subscribe().begin(), + request.resource_names_subscribe().end()}; + std::set actual_unsub{request.resource_names_unsubscribe().begin(), + request.resource_names_unsubscribe().end()}; + auto sub_result = compareSets(expected_sub, actual_sub, "expected_resource_subscriptions"); + if (!sub_result) { + return sub_result; + } + auto unsub_result = + compareSets(expected_unsub, actual_unsub, "expected_resource_unsubscriptions"); + if (!unsub_result) { + return unsub_result; + } + // (We don't care about response_nonce or initial_resource_versions.) + + if (request.error_detail().code() != expected_error_code) { return AssertionFailure() << fmt::format( - "newly UNsubscribed resources {} do not match expected {} in {}", - fmt::join(resource_unsubscriptions.begin(), resource_unsubscriptions.end(), ","), - fmt::join(expected_resource_unsubscriptions.begin(), - expected_resource_unsubscriptions.end(), ","), - request.DebugString()); + "error code {} does not match expected {}. (Error message is {}).", + request.error_detail().code(), expected_error_code, + request.error_detail().message()); + } + if (expected_error_code != Grpc::Status::GrpcStatus::Ok && + request.error_detail().message().find(expected_error_substring) == std::string::npos) { + return AssertionFailure() << "\"" << expected_error_substring + << "\" is not a substring of actual error message \"" + << request.error_detail().message() << "\""; } return AssertionSuccess(); } diff --git a/test/integration/integration.h b/test/integration/integration.h index 7cfa3b726435..7cd2109cc859 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -220,12 +220,14 @@ class BaseIntegrationTest : Logger::Loggable { // sending/receiving to/from the (imaginary) xDS server. You should almost always use // compareDiscoveryRequest() and sendDiscoveryResponse(), but the SotW/delta-specific versions are // available if you're writing a SotW/delta-specific test. + // TODO(fredlas) expect_node was defaulting false here; the delta+SotW unification work restores + // it. AssertionResult compareDiscoveryRequest(const std::string& expected_type_url, const std::string& expected_version, const std::vector& expected_resource_names, const std::vector& expected_resource_names_added, const std::vector& expected_resource_names_removed, - bool expect_node = false, + bool expect_node = true, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_message = ""); template @@ -256,9 +258,12 @@ class BaseIntegrationTest : Logger::Loggable { const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& stream, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_message = ""); + + // TODO(fredlas) expect_node was defaulting false here; the delta+SotW unification work restores + // it. AssertionResult compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, - const std::vector& expected_resource_names, bool expect_node = false, + const std::vector& expected_resource_names, bool expect_node = true, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_message = ""); @@ -271,8 +276,11 @@ class BaseIntegrationTest : Logger::Loggable { for (const auto& message : messages) { discovery_response.add_resources()->PackFrom(message); } + static int next_nonce_counter = 0; + discovery_response.set_nonce(absl::StrCat("nonce", next_nonce_counter++)); xds_stream_->sendGrpcMessage(discovery_response); } + template void sendDeltaDiscoveryResponse(const std::string& type_url, const std::vector& added_or_updated, @@ -296,7 +304,8 @@ class BaseIntegrationTest : Logger::Loggable { resource->mutable_resource()->PackFrom(message); } *response.mutable_removed_resources() = {removed.begin(), removed.end()}; - response.set_nonce("noncense"); + static int next_nonce_counter = 0; + response.set_nonce(absl::StrCat("nonce", next_nonce_counter++)); stream->sendGrpcMessage(response); } diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index 02283b3927c2..6f223f1a626d 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -6,6 +6,8 @@ namespace Envoy { namespace { +// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification +// work restores it here. std::string tdsBootstrapConfig(absl::string_view api_type) { return fmt::format(R"EOF( static_resources: @@ -36,7 +38,7 @@ std::string tdsBootstrapConfig(absl::string_view api_type) { grpc_services: envoy_grpc: cluster_name: rtds_cluster - set_node_on_first_message_only: true + set_node_on_first_message_only: false - name: some_admin_layer admin_layer: {{}} admin: @@ -110,7 +112,8 @@ class RtdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public H uint32_t initial_keys_{}; }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, RtdsIntegrationTest, DELTA_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, RtdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); TEST_P(RtdsIntegrationTest, RtdsReload) { initialize(); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 5f4bb27395e4..3caca65ec896 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -13,7 +13,7 @@ namespace Envoy { namespace { class ScopedRdsIntegrationTest : public HttpIntegrationTest, - public Grpc::DeltaSotwGrpcClientIntegrationParamTest { + public Grpc::DeltaSotwIntegrationParamTest { protected: struct FakeUpstreamInfo { FakeHttpConnectionPtr connection_; @@ -217,6 +217,8 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, response); } + bool isDelta() { return sotwOrDelta() == Grpc::SotwOrDelta::Delta; } + const std::string srds_config_name_{"foo-scoped-routes"}; FakeUpstreamInfo scoped_rds_upstream_info_; FakeUpstreamInfo rds_upstream_info_; diff --git a/test/mocks/config/BUILD b/test/mocks/config/BUILD index 7cd7830ad794..69698c7dc732 100644 --- a/test/mocks/config/BUILD +++ b/test/mocks/config/BUILD @@ -16,7 +16,6 @@ envoy_cc_mock( "//include/envoy/config:config_provider_manager_interface", "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", - "//include/envoy/config:xds_grpc_context_interface", "//source/common/config:config_provider_lib", "//source/common/config:resources_lib", "//source/common/protobuf:utility_lib", diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index 4a3e3099e0c1..7d2846e44a6d 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -10,15 +10,15 @@ namespace Envoy { namespace Config { MockSubscriptionFactory::MockSubscriptionFactory() { - ON_CALL(*this, subscriptionFromConfigSource(_, _, _, _)) - .WillByDefault(testing::Invoke([this](const envoy::api::v2::core::ConfigSource&, - absl::string_view, Stats::Scope&, - SubscriptionCallbacks& callbacks) -> SubscriptionPtr { - auto ret = std::make_unique>(); - subscription_ = ret.get(); - callbacks_ = &callbacks; - return ret; - })); + ON_CALL(*this, subscriptionFromConfigSource(_, _, _, _, _)) + .WillByDefault(testing::Invoke( + [this](const envoy::api::v2::core::ConfigSource&, absl::string_view, Stats::Scope&, + SubscriptionCallbacks& callbacks, bool) -> SubscriptionPtr { + auto ret = std::make_unique>(); + subscription_ = ret.get(); + callbacks_ = &callbacks; + return ret; + })); ON_CALL(*this, messageValidationVisitor()) .WillByDefault(testing::ReturnRef(ProtobufMessage::getStrictValidationVisitor())); } diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index b579cb56b722..7b88cea67c88 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -4,7 +4,6 @@ #include "envoy/config/config_provider_manager.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" -#include "envoy/config/xds_grpc_context.h" #include "common/config/config_provider_impl.h" #include "common/config/resources.h" @@ -45,7 +44,7 @@ template class MockSubscriptionCallbacks : public Subscript class MockSubscription : public Subscription { public: MOCK_METHOD1(start, void(const std::set& resources)); - MOCK_METHOD1(updateResources, void(const std::set& update_to_these_names)); + MOCK_METHOD1(updateResourceInterest, void(const std::set& update_to_these_names)); }; class MockSubscriptionFactory : public SubscriptionFactory { @@ -53,10 +52,10 @@ class MockSubscriptionFactory : public SubscriptionFactory { MockSubscriptionFactory(); ~MockSubscriptionFactory() override; - MOCK_METHOD4(subscriptionFromConfigSource, + MOCK_METHOD5(subscriptionFromConfigSource, SubscriptionPtr(const envoy::api::v2::core::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, - SubscriptionCallbacks& callbacks)); + SubscriptionCallbacks& callbacks, bool is_delta)); MOCK_METHOD0(messageValidationVisitor, ProtobufMessage::ValidationVisitor&()); MockSubscription* subscription_{}; @@ -85,6 +84,19 @@ class MockGrpcMux : public GrpcMux { MOCK_METHOD1(pause, void(const std::string& type_url)); MOCK_METHOD1(resume, void(const std::string& type_url)); MOCK_CONST_METHOD1(paused, bool(const std::string& type_url)); + + MOCK_METHOD5(addSubscription, + void(const std::set& resources, const std::string& type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats& stats, + std::chrono::milliseconds init_fetch_timeout)); + MOCK_METHOD2(updateResourceInterest, + void(const std::set& resources, const std::string& type_url)); + + MOCK_METHOD5(addOrUpdateWatch, + Watch*(const std::string& type_url, Watch* watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout)); + MOCK_METHOD2(removeWatch, void(const std::string& type_url, Watch* watch)); }; class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index e1416736308f..ba07f3c6f97c 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -406,11 +406,11 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { MockRouteConfigProviderManager(); ~MockRouteConfigProviderManager() override; - MOCK_METHOD4(createRdsRouteConfigProvider, + MOCK_METHOD5(createRdsRouteConfigProvider, RouteConfigProviderPtr( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix, Init::Manager& init_manager)); + const std::string& stat_prefix, Init::Manager& init_manager, bool is_delta)); MOCK_METHOD2(createStaticRouteConfigProvider, RouteConfigProviderPtr(const envoy::api::v2::RouteConfiguration& route_config, Server::Configuration::FactoryContext& factory_context)); diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index bcbc232bd240..1487bc0b8852 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -234,11 +234,13 @@ class MockListenerComponentFactory : public ListenerComponentFactory { DrainManagerPtr createDrainManager(envoy::api::v2::Listener::DrainType drain_type) override { return DrainManagerPtr{createDrainManager_(drain_type)}; } - LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { - return LdsApiPtr{createLdsApi_(lds_config)}; + LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) override { + return LdsApiPtr{createLdsApi_(lds_config, is_delta)}; } - MOCK_METHOD1(createLdsApi_, LdsApi*(const envoy::api::v2::core::ConfigSource& lds_config)); + MOCK_METHOD2(createLdsApi_, + LdsApi*(const envoy::api::v2::core::ConfigSource& lds_config, bool is_delta)); MOCK_METHOD2(createNetworkFilterFactoryList, std::vector( const Protobuf::RepeatedPtrField& filters, @@ -269,7 +271,8 @@ class MockListenerManager : public ListenerManager { MOCK_METHOD3(addOrUpdateListener, bool(const envoy::api::v2::Listener& config, const std::string& version_info, bool modifiable)); - MOCK_METHOD1(createLdsApi, void(const envoy::api::v2::core::ConfigSource& lds_config)); + MOCK_METHOD2(createLdsApi, + void(const envoy::api::v2::core::ConfigSource& lds_config, bool is_delta)); MOCK_METHOD0(listeners, std::vector>()); MOCK_METHOD0(numConnections, uint64_t()); MOCK_METHOD1(removeListener, bool(const std::string& listener_name)); diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 4141474ec23e..e5843f0bc64b 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -139,7 +139,7 @@ MockClusterManager::MockClusterManager() { ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault(ReturnRef(async_client_)); ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault((ReturnRef(async_client_))); ON_CALL(*this, bindConfig()).WillByDefault(ReturnRef(bind_config_)); - ON_CALL(*this, adsMux()).WillByDefault(ReturnRef(ads_mux_)); + ON_CALL(*this, adsMux()).WillByDefault(Return(ads_mux_)); ON_CALL(*this, grpcAsyncClientManager()).WillByDefault(ReturnRef(async_client_manager_)); ON_CALL(*this, localClusterName()).WillByDefault((ReturnRef(local_cluster_name_))); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 66dcb05380c1..39e2be2bb5a2 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -267,8 +267,8 @@ class MockClusterManagerFactory : public ClusterManagerFactory { const envoy::api::v2::Cluster& cluster, ClusterManager& cm, Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api)); - MOCK_METHOD2(createCds, - CdsApiPtr(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm)); + MOCK_METHOD3(createCds, CdsApiPtr(const envoy::api::v2::core::ConfigSource& cds_config, + bool is_delta, ClusterManager& cm)); private: NiceMock secret_manager_; @@ -319,13 +319,14 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD1(removeCluster, bool(const std::string& cluster)); MOCK_METHOD0(shutdown, void()); MOCK_CONST_METHOD0(bindConfig, const envoy::api::v2::core::BindConfig&()); - MOCK_METHOD0(adsMux, Config::GrpcMux&()); + MOCK_METHOD0(adsMux, Config::GrpcMuxSharedPtr()); MOCK_METHOD0(grpcAsyncClientManager, Grpc::AsyncClientManager&()); MOCK_CONST_METHOD0(versionInfo, const std::string()); MOCK_CONST_METHOD0(localClusterName, const std::string&()); MOCK_METHOD1(addThreadLocalClusterUpdateCallbacks_, ClusterUpdateCallbacksHandle*(ClusterUpdateCallbacks& callbacks)); MOCK_CONST_METHOD0(warmingClusterCount, std::size_t()); + MOCK_CONST_METHOD0(xdsIsDelta, bool()); MOCK_METHOD0(subscriptionFactory, Config::SubscriptionFactory&()); NiceMock conn_pool_; @@ -333,7 +334,7 @@ class MockClusterManager : public ClusterManager { NiceMock tcp_conn_pool_; NiceMock thread_local_cluster_; envoy::api::v2::core::BindConfig bind_config_; - NiceMock ads_mux_; + std::shared_ptr> ads_mux_; NiceMock async_client_manager_; std::string local_cluster_name_; NiceMock cluster_manager_factory_; diff --git a/test/server/lds_api_test.cc b/test/server/lds_api_test.cc index a5ffe1378ace..72aee52328b5 100644 --- a/test/server/lds_api_test.cc +++ b/test/server/lds_api_test.cc @@ -36,7 +36,7 @@ class LdsApiTest : public testing::Test { envoy::api::v2::core::ConfigSource lds_config; EXPECT_CALL(init_manager_, add(_)); lds_ = std::make_unique(lds_config, cluster_manager_, init_manager_, store_, - listener_manager_, validation_visitor_); + listener_manager_, validation_visitor_, false); EXPECT_CALL(*cluster_manager_.subscription_factory_.subscription_, start(_)); init_target_handle_->initialize(init_watcher_); lds_callbacks_ = cluster_manager_.subscription_factory_.callbacks_; @@ -83,6 +83,7 @@ class LdsApiTest : public testing::Test { listeners.Add()->PackFrom(listener); } + std::shared_ptr> grpc_mux_; NiceMock cluster_manager_; Init::MockManager init_manager_; Init::ExpectableWatcherImpl init_watcher_; diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 12c436b8ad60..91882a64f73f 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -590,9 +590,9 @@ TEST_F(ListenerManagerImplTest, AddOrUpdateListener) { InSequence s; auto* lds_api = new MockLdsApi(); - EXPECT_CALL(listener_factory_, createLdsApi_(_)).WillOnce(Return(lds_api)); + EXPECT_CALL(listener_factory_, createLdsApi_(_, _)).WillOnce(Return(lds_api)); envoy::api::v2::core::ConfigSource lds_config; - manager_->createLdsApi(lds_config); + manager_->createLdsApi(lds_config, false); EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("")); checkConfigDump(R"EOF( diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 9204d82d1893..ab5842d2a194 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -330,6 +330,7 @@ abcd absl accessor accessors +acks acls addr agg @@ -644,6 +645,7 @@ paren parentid parsers passthroughs +pausable pcall pcap pclose @@ -818,6 +820,7 @@ unordered unowned unparented unpause +unpaused unpopulated unprotect unref From 57ac960c54389dfa55df1d5c7db2754e3e326ffd Mon Sep 17 00:00:00 2001 From: danzh Date: Wed, 2 Oct 2019 00:22:59 -0400 Subject: [PATCH 30/32] quiche: modify some envoy quic implementation for code sharing (#8349) Signed-off-by: Dan Zhang --- source/common/network/BUILD | 1 + source/common/network/udp_listener_impl.cc | 66 ++------- source/common/network/udp_listener_impl.h | 11 ++ source/common/network/utility.cc | 64 +++++++++ source/common/network/utility.h | 56 ++++++++ source/extensions/quic_listeners/quiche/BUILD | 43 ++++-- .../quiche/active_quic_listener.cc | 3 +- .../quic_listeners/quiche/codec_impl.cc | 8 +- .../quic_listeners/quiche/codec_impl.h | 13 +- .../quiche/envoy_quic_connection.cc | 58 ++------ .../quiche/envoy_quic_connection.h | 31 ++--- .../quiche/envoy_quic_dispatcher.cc | 9 +- .../quiche/envoy_quic_packet_writer.cc | 16 ++- .../quiche/envoy_quic_packet_writer.h | 5 +- .../quiche/envoy_quic_server_connection.cc | 61 +++++++++ .../quiche/envoy_quic_server_connection.h | 34 +++++ .../quiche/envoy_quic_server_session.cc | 115 ++-------------- .../quiche/envoy_quic_server_session.h | 98 +------------- .../quic_filter_manager_connection_impl.cc | 116 ++++++++++++++++ .../quic_filter_manager_connection_impl.h | 127 ++++++++++++++++++ test/extensions/quic_listeners/quiche/BUILD | 7 +- .../quiche/active_quic_listener_test.cc | 1 + .../quiche/envoy_quic_dispatcher_test.cc | 1 + .../quiche/envoy_quic_server_session_test.cc | 47 +++---- .../quiche/envoy_quic_server_stream_test.cc | 9 +- .../quiche/envoy_quic_writer_test.cc | 98 +++++++------- 26 files changed, 665 insertions(+), 433 deletions(-) create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_server_connection.cc create mode 100644 source/extensions/quic_listeners/quiche/envoy_quic_server_connection.h create mode 100644 source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.cc create mode 100644 source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h diff --git a/source/common/network/BUILD b/source/common/network/BUILD index b43a9ad1e8a7..be7a5f610dad 100644 --- a/source/common/network/BUILD +++ b/source/common/network/BUILD @@ -257,6 +257,7 @@ envoy_cc_library( "//include/envoy/network:connection_interface", "//include/envoy/stats:stats_interface", "//source/common/api:os_sys_calls_lib", + "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:cleanup_lib", "//source/common/common:utility_lib", diff --git a/source/common/network/udp_listener_impl.cc b/source/common/network/udp_listener_impl.cc index 290511b29843..a3c5237e60f1 100644 --- a/source/common/network/udp_listener_impl.cc +++ b/source/common/network/udp_listener_impl.cc @@ -27,9 +27,6 @@ namespace Envoy { namespace Network { -// Max UDP payload. -static const uint64_t MAX_UDP_PACKET_SIZE = 1500; - UdpListenerImpl::UdpListenerImpl(Event::DispatcherImpl& dispatcher, Socket& socket, UdpListenerCallbacks& cb, TimeSource& time_source) : BaseListenerImpl(dispatcher, socket), cb_(cb), time_source_(time_source) { @@ -72,19 +69,11 @@ void UdpListenerImpl::onSocketEvent(short flags) { void UdpListenerImpl::handleReadCallback() { ENVOY_UDP_LOG(trace, "handleReadCallback"); - // TODO(danzh) make this variable configurable to support jumbo frames. - const uint64_t read_buffer_length = MAX_UDP_PACKET_SIZE; do { - Buffer::InstancePtr buffer = std::make_unique(); - Buffer::RawSlice slice; - const uint64_t num_slices = buffer->reserve(read_buffer_length, &slice, 1); - ASSERT(num_slices == 1); - - IoHandle::RecvMsgOutput output(&packets_dropped_); uint32_t old_packets_dropped = packets_dropped_; MonotonicTime receive_time = time_source_.monotonicTime(); - Api::IoCallUint64Result result = socket_.ioHandle().recvmsg( - &slice, num_slices, socket_.localAddress()->ip()->port(), output); + Api::IoCallUint64Result result = + Utility::readFromSocket(socket_, *this, receive_time, &packets_dropped_); if (!result.ok()) { // No more to read or encountered a system error. @@ -104,8 +93,6 @@ void UdpListenerImpl::handleReadCallback() { ENVOY_UDP_LOG(trace, "received 0-length packet"); } - RELEASE_ASSERT(output.local_address_ != nullptr, "fail to get local address from IP header"); - if (packets_dropped_ != old_packets_dropped) { // The kernel tracks SO_RXQ_OVFL as a uint32 which can overflow to a smaller // value. So as long as this count differs from previously recorded value, @@ -118,30 +105,17 @@ void UdpListenerImpl::handleReadCallback() { ENVOY_UDP_LOG(debug, "Kernel dropped {} more packets. Consider increase receive buffer size.", delta); } - - // Adjust used memory length. - slice.len_ = std::min(slice.len_, static_cast(result.rc_)); - buffer->commit(&slice, 1); - - ENVOY_UDP_LOG(trace, "recvmsg bytes {}", result.rc_); - - RELEASE_ASSERT(output.peer_address_ != nullptr, - fmt::format("Unable to get remote address for fd: {}, local address: {} ", - socket_.ioHandle().fd(), socket_.localAddress()->asString())); - - // Unix domain sockets are not supported - RELEASE_ASSERT(output.peer_address_->type() == Address::Type::Ip, - fmt::format("Unsupported remote address: {} local address: {}, receive size: " - "{}", - output.peer_address_->asString(), socket_.localAddress()->asString(), - result.rc_)); - - UdpRecvData recvData{std::move(output.local_address_), std::move(output.peer_address_), - std::move(buffer), receive_time}; - cb_.onData(recvData); } while (true); } +void UdpListenerImpl::processPacket(Address::InstanceConstSharedPtr local_address, + Address::InstanceConstSharedPtr peer_address, + Buffer::InstancePtr buffer, MonotonicTime receive_time) { + UdpRecvData recvData{std::move(local_address), std::move(peer_address), std::move(buffer), + receive_time}; + cb_.onData(recvData); +} + void UdpListenerImpl::handleWriteCallback() { ENVOY_UDP_LOG(trace, "handleWriteCallback"); cb_.onWriteReady(socket_); @@ -159,28 +133,12 @@ Api::IoCallUint64Result UdpListenerImpl::send(const UdpSendData& send_data) { uint64_t num_slices = buffer.getRawSlices(nullptr, 0); STACK_ARRAY(slices, Buffer::RawSlice, num_slices); buffer.getRawSlices(slices.begin(), num_slices); - Api::IoCallUint64Result send_result( - /*rc=*/0, /*err=*/Api::IoErrorPtr(nullptr, IoSocketError::deleteIoError)); - do { - send_result = socket_.ioHandle().sendmsg(slices.begin(), num_slices, 0, send_data.local_ip_, - send_data.peer_address_); - } while (!send_result.ok() && - // Send again if interrupted. - send_result.err_->getErrorCode() == Api::IoError::IoErrorCode::Interrupt); - - if (send_result.ok()) { - ASSERT(send_result.rc_ == buffer.length()); - ENVOY_UDP_LOG(trace, "sendmsg sent:{} bytes", send_result.rc_); - } else { - ENVOY_UDP_LOG(debug, "sendmsg failed with error code {}: {}", - static_cast(send_result.err_->getErrorCode()), - send_result.err_->getErrorDetails()); - } + Api::IoCallUint64Result send_result = Utility::writeToSocket( + socket_, slices.begin(), num_slices, send_data.local_ip_, send_data.peer_address_); // The send_result normalizes the rc_ value to 0 in error conditions. // The drain call is hence 'safe' in success and failure cases. buffer.drain(send_result.rc_); - return send_result; } diff --git a/source/common/network/udp_listener_impl.h b/source/common/network/udp_listener_impl.h index b526b8034f8d..c24d1cc14134 100644 --- a/source/common/network/udp_listener_impl.h +++ b/source/common/network/udp_listener_impl.h @@ -7,6 +7,7 @@ #include "common/buffer/buffer_impl.h" #include "common/event/event_impl_base.h" #include "common/event/file_event_impl.h" +#include "common/network/utility.h" #include "base_listener_impl.h" @@ -18,6 +19,7 @@ namespace Network { */ class UdpListenerImpl : public BaseListenerImpl, public virtual UdpListener, + public UdpPacketProcessor, protected Logger::Loggable { public: UdpListenerImpl(Event::DispatcherImpl& dispatcher, Socket& socket, UdpListenerCallbacks& cb, @@ -34,6 +36,15 @@ class UdpListenerImpl : public BaseListenerImpl, const Address::InstanceConstSharedPtr& localAddress() const override; Api::IoCallUint64Result send(const UdpSendData& data) override; + void processPacket(Address::InstanceConstSharedPtr local_address, + Address::InstanceConstSharedPtr peer_address, Buffer::InstancePtr buffer, + MonotonicTime receive_time) override; + + uint64_t maxPacketSize() const override { + // TODO(danzh) make this variable configurable to support jumbo frames. + return MAX_UDP_PACKET_SIZE; + } + protected: void handleWriteCallback(); void handleReadCallback(); diff --git a/source/common/network/utility.cc b/source/common/network/utility.cc index 638547cd9d14..667d0ccc997f 100644 --- a/source/common/network/utility.cc +++ b/source/common/network/utility.cc @@ -28,8 +28,10 @@ #include "common/common/assert.h" #include "common/common/cleanup.h" #include "common/common/utility.h" +#include "common/buffer/buffer_impl.h" #include "common/network/address_impl.h" #include "common/protobuf/protobuf.h" +#include "common/network/io_socket_error_impl.h" #include "common/common/fmt.h" @@ -506,5 +508,67 @@ Utility::protobufAddressSocketType(const envoy::api::v2::core::Address& proto_ad } } +Api::IoCallUint64Result Utility::writeToSocket(Network::Socket& socket, Buffer::RawSlice* slices, + uint64_t num_slices, const Address::Ip* local_ip, + const Address::Instance& peer_address) { + Api::IoCallUint64Result send_result( + /*rc=*/0, /*err=*/Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)); + do { + send_result = socket.ioHandle().sendmsg(slices, num_slices, 0, local_ip, peer_address); + } while (!send_result.ok() && + // Send again if interrupted. + send_result.err_->getErrorCode() == Api::IoError::IoErrorCode::Interrupt); + + if (send_result.ok()) { + ENVOY_LOG_MISC(trace, "sendmsg sent:{} bytes", send_result.rc_); + } else { + ENVOY_LOG_MISC(debug, "sendmsg failed with error code {}: {}", + static_cast(send_result.err_->getErrorCode()), + send_result.err_->getErrorDetails()); + } + return send_result; +} + +Api::IoCallUint64Result Utility::readFromSocket(Network::Socket& socket, + UdpPacketProcessor& udp_packet_processor, + MonotonicTime receive_time, + uint32_t* packets_dropped) { + Buffer::InstancePtr buffer = std::make_unique(); + Buffer::RawSlice slice; + const uint64_t num_slices = buffer->reserve(udp_packet_processor.maxPacketSize(), &slice, 1); + ASSERT(num_slices == 1); + + IoHandle::RecvMsgOutput output(packets_dropped); + Api::IoCallUint64Result result = + socket.ioHandle().recvmsg(&slice, num_slices, socket.localAddress()->ip()->port(), output); + + if (!result.ok()) { + return result; + } + + RELEASE_ASSERT(output.local_address_ != nullptr, "fail to get local address from IP header"); + + // Adjust used memory length. + slice.len_ = std::min(slice.len_, static_cast(result.rc_)); + buffer->commit(&slice, 1); + + ENVOY_LOG_MISC(trace, "recvmsg bytes {}", result.rc_); + + RELEASE_ASSERT(output.peer_address_ != nullptr, + fmt::format("Unable to get remote address for fd: {}, local address: {} ", + socket.ioHandle().fd(), socket.localAddress()->asString())); + + // Unix domain sockets are not supported + RELEASE_ASSERT(output.peer_address_->type() == Address::Type::Ip, + fmt::format("Unsupported remote address: {} local address: {}, receive size: " + "{}", + output.peer_address_->asString(), socket.localAddress()->asString(), + result.rc_)); + udp_packet_processor.processPacket(std::move(output.local_address_), + std::move(output.peer_address_), std::move(buffer), + receive_time); + return result; +} + } // namespace Network } // namespace Envoy diff --git a/source/common/network/utility.h b/source/common/network/utility.h index 24eb526b6a6e..53d3db2c75e6 100644 --- a/source/common/network/utility.h +++ b/source/common/network/utility.h @@ -28,6 +28,35 @@ class PortRange { using PortRangeList = std::list; +/** + * A callback interface used by readFromSocket() to pass packets read from + * socket. + */ +class UdpPacketProcessor { +public: + virtual ~UdpPacketProcessor() = default; + + /** + * Consume the packet read out of the socket with the information from UDP + * header. + * @param local_address is the destination address in the UDP header. + * @param peer_address is the source address in the UDP header. + * @param buffer contains the packet read. + * @param receive_time is the time when the packet is read. + */ + virtual void processPacket(Address::InstanceConstSharedPtr local_address, + Address::InstanceConstSharedPtr peer_address, + Buffer::InstancePtr buffer, MonotonicTime receive_time) PURE; + + /** + * The expected max size of the packet to be read. If it's smaller than + * actually packets received, the payload will be truncated. + */ + virtual uint64_t maxPacketSize() const PURE; +}; + +static const uint64_t MAX_UDP_PACKET_SIZE = 1500; + /** * Common network utility routines. */ @@ -258,6 +287,33 @@ class Utility { static Address::SocketType protobufAddressSocketType(const envoy::api::v2::core::Address& proto_address); + /** + * Send a packet via given UDP socket with specific source address. + * @param socket is the UDP socket used to send. + * @param slices points to the buffers containing the packet. + * @param num_slices is the number of buffers. + * @param local_ip is the source address to be used to send. + * @param peer_address is the destination address to send to. + */ + static Api::IoCallUint64Result writeToSocket(Network::Socket& socket, Buffer::RawSlice* slices, + uint64_t num_slices, const Address::Ip* local_ip, + const Address::Instance& peer_address); + + /** + * Read a packet from given UDP socket and pass the packet to given + * UdpPacketProcessor. + * @param socket is the UDP socket to read from. + * @param udp_packet_processor is the callback to receive the packet. + * @param receive_time is the timestamp passed to udp_packet_processor for the + * receive time of the packet. + * @param packets_dropped is the output parameter for number of packets dropped in kernel. If the + * caller is not interested in it, nullptr can be passed in. + */ + static Api::IoCallUint64Result readFromSocket(Network::Socket& socket, + UdpPacketProcessor& udp_packet_processor, + MonotonicTime receive_time, + uint32_t* packets_dropped); + private: static void throwWithMalformedIp(const std::string& ip_address); diff --git a/source/extensions/quic_listeners/quiche/BUILD b/source/extensions/quic_listeners/quiche/BUILD index 3e059d44d795..d0e569e3d93d 100644 --- a/source/extensions/quic_listeners/quiche/BUILD +++ b/source/extensions/quic_listeners/quiche/BUILD @@ -118,7 +118,7 @@ envoy_cc_library( ) envoy_cc_library( - name = "codec_impl_lib", + name = "codec_lib", srcs = ["codec_impl.cc"], hdrs = ["codec_impl.h"], tags = ["nofips"], @@ -130,18 +130,28 @@ envoy_cc_library( ) envoy_cc_library( - name = "envoy_quic_server_session_lib", - srcs = ["envoy_quic_server_session.cc"], - hdrs = ["envoy_quic_server_session.h"], + name = "quic_filter_manager_connection_lib", + srcs = ["quic_filter_manager_connection_impl.cc"], + hdrs = ["quic_filter_manager_connection_impl.h"], tags = ["nofips"], deps = [ ":envoy_quic_connection_lib", - ":envoy_quic_server_stream_lib", "//include/envoy/event:dispatcher_interface", "//include/envoy/network:connection_interface", "//source/common/common:empty_string", "//source/common/network:filter_manager_lib", "//source/common/stream_info:stream_info_lib", + ], +) + +envoy_cc_library( + name = "envoy_quic_server_session_lib", + srcs = ["envoy_quic_server_session.cc"], + hdrs = ["envoy_quic_server_session.h"], + tags = ["nofips"], + deps = [ + ":envoy_quic_server_stream_lib", + ":quic_filter_manager_connection_lib", "@com_googlesource_quiche//:quic_core_http_spdy_session_lib", ], ) @@ -165,23 +175,30 @@ envoy_cc_library( "//include/envoy/network:connection_interface", "//source/common/network:listen_socket_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_utils_lib", - "//source/server:connection_handler_lib", + "//source/extensions/transport_sockets:well_known_names", "@com_googlesource_quiche//:quic_core_connection_lib", ], ) envoy_cc_library( - name = "envoy_quic_dispatcher_lib", - srcs = [ - "envoy_quic_dispatcher.cc", - ], - hdrs = [ - "envoy_quic_dispatcher.h", - ], + name = "envoy_quic_server_connection_lib", + srcs = ["envoy_quic_server_connection.cc"], + hdrs = ["envoy_quic_server_connection.h"], tags = ["nofips"], deps = [ ":envoy_quic_connection_lib", + "//source/server:connection_handler_lib", + ], +) + +envoy_cc_library( + name = "envoy_quic_dispatcher_lib", + srcs = ["envoy_quic_dispatcher.cc"], + hdrs = ["envoy_quic_dispatcher.h"], + tags = ["nofips"], + deps = [ ":envoy_quic_proof_source_lib", + ":envoy_quic_server_connection_lib", ":envoy_quic_server_session_lib", "//include/envoy/network:listener_interface", "//source/server:connection_handler_lib", diff --git a/source/extensions/quic_listeners/quiche/active_quic_listener.cc b/source/extensions/quic_listeners/quiche/active_quic_listener.cc index 6272cc98df15..15d1b700e0b4 100644 --- a/source/extensions/quic_listeners/quiche/active_quic_listener.cc +++ b/source/extensions/quic_listeners/quiche/active_quic_listener.cc @@ -23,7 +23,8 @@ ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher, Network::UdpListenerPtr&& listener, Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config) - : ActiveQuicListener(dispatcher, parent, std::make_unique(*listener), + : ActiveQuicListener(dispatcher, parent, + std::make_unique(listener_config.socket()), std::move(listener), listener_config, quic_config) {} ActiveQuicListener::ActiveQuicListener(Event::Dispatcher& dispatcher, diff --git a/source/extensions/quic_listeners/quiche/codec_impl.cc b/source/extensions/quic_listeners/quiche/codec_impl.cc index 4d9846ac6b26..fdb060cb7c15 100644 --- a/source/extensions/quic_listeners/quiche/codec_impl.cc +++ b/source/extensions/quic_listeners/quiche/codec_impl.cc @@ -3,10 +3,6 @@ namespace Envoy { namespace Quic { -void QuicHttpConnectionImplBase::goAway() { - quic_session_.SendGoAway(quic::QUIC_PEER_GOING_AWAY, "server shutdown imminent"); -} - bool QuicHttpConnectionImplBase::wantsToWrite() { return quic_session_.HasDataToWrite(); } // TODO(danzh): modify QUIC stack to react based on aggregated bytes across all @@ -19,5 +15,9 @@ void QuicHttpConnectionImplBase::onUnderlyingConnectionBelowWriteBufferLowWaterm NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } +void QuicHttpServerConnectionImpl::goAway() { + quic_server_session_.SendGoAway(quic::QUIC_PEER_GOING_AWAY, "server shutdown imminent"); +} + } // namespace Quic } // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/codec_impl.h b/source/extensions/quic_listeners/quiche/codec_impl.h index 4920b50a0247..debff738cb04 100644 --- a/source/extensions/quic_listeners/quiche/codec_impl.h +++ b/source/extensions/quic_listeners/quiche/codec_impl.h @@ -14,7 +14,7 @@ namespace Quic { class QuicHttpConnectionImplBase : public virtual Http::Connection, protected Logger::Loggable { public: - QuicHttpConnectionImplBase(EnvoyQuicServerSession& quic_session) : quic_session_(quic_session) {} + QuicHttpConnectionImplBase(quic::QuicSpdySession& quic_session) : quic_session_(quic_session) {} // Http::Connection void dispatch(Buffer::Instance& /*data*/) override { @@ -27,7 +27,6 @@ class QuicHttpConnectionImplBase : public virtual Http::Connection, // TODO(danzh) add Http3 enum value for QUIC. return Http::Protocol::Http2; } - void goAway() override; // Returns true if the session has data to send but queued in connection or // stream send buffer. bool wantsToWrite() override; @@ -35,7 +34,7 @@ class QuicHttpConnectionImplBase : public virtual Http::Connection, void onUnderlyingConnectionBelowWriteBufferLowWatermark() override; protected: - EnvoyQuicServerSession& quic_session_; + quic::QuicSpdySession& quic_session_; }; class QuicHttpServerConnectionImpl : public QuicHttpConnectionImplBase, @@ -43,15 +42,19 @@ class QuicHttpServerConnectionImpl : public QuicHttpConnectionImplBase, public: QuicHttpServerConnectionImpl(EnvoyQuicServerSession& quic_session, Http::ServerConnectionCallbacks& callbacks) - : QuicHttpConnectionImplBase(quic_session) { + : QuicHttpConnectionImplBase(quic_session), quic_server_session_(quic_session) { quic_session.setHttpConnectionCallbacks(callbacks); } // Http::Connection + void goAway() override; void shutdownNotice() override { // TODO(danzh): Add double-GOAWAY support in QUIC. - ENVOY_CONN_LOG(error, "Shutdown notice is not propagated to QUIC.", quic_session_); + ENVOY_CONN_LOG(error, "Shutdown notice is not propagated to QUIC.", quic_server_session_); } + +private: + EnvoyQuicServerSession& quic_server_session_; }; } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_connection.cc b/source/extensions/quic_listeners/quiche/envoy_quic_connection.cc index 95e97e5f6966..dcc311a6eaac 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_connection.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_connection.cc @@ -1,59 +1,25 @@ #include "extensions/quic_listeners/quiche/envoy_quic_connection.h" -#include "common/network/listen_socket_impl.h" - #include "extensions/quic_listeners/quiche/envoy_quic_utils.h" -#include "extensions/quic_listeners/quiche/quic_io_handle_wrapper.h" -#include "extensions/transport_sockets/well_known_names.h" namespace Envoy { namespace Quic { -EnvoyQuicConnection::EnvoyQuicConnection( - const quic::QuicConnectionId& server_connection_id, - quic::QuicSocketAddress initial_peer_address, quic::QuicConnectionHelperInterface& helper, - quic::QuicAlarmFactory& alarm_factory, quic::QuicPacketWriter& writer, bool owns_writer, - quic::Perspective perspective, const quic::ParsedQuicVersionVector& supported_versions, - Network::ListenerConfig& listener_config, Server::ListenerStats& listener_stats) +EnvoyQuicConnection::EnvoyQuicConnection(const quic::QuicConnectionId& server_connection_id, + quic::QuicSocketAddress initial_peer_address, + quic::QuicConnectionHelperInterface& helper, + quic::QuicAlarmFactory& alarm_factory, + quic::QuicPacketWriter* writer, bool owns_writer, + quic::Perspective perspective, + const quic::ParsedQuicVersionVector& supported_versions, + Network::ConnectionSocketPtr&& connection_socket) : quic::QuicConnection(server_connection_id, initial_peer_address, &helper, &alarm_factory, - &writer, owns_writer, perspective, supported_versions), - listener_config_(listener_config), listener_stats_(listener_stats) {} - -bool EnvoyQuicConnection::OnPacketHeader(const quic::QuicPacketHeader& header) { - if (quic::QuicConnection::OnPacketHeader(header) && connection_socket_ == nullptr) { - ASSERT(self_address().IsInitialized()); - // Self address should be initialized by now. It's time to install filters. - Network::Address::InstanceConstSharedPtr local_addr = - quicAddressToEnvoyAddressInstance(self_address()); + writer, owns_writer, perspective, supported_versions), + connection_socket_(std::move(connection_socket)) {} - Network::Address::InstanceConstSharedPtr remote_addr = - quicAddressToEnvoyAddressInstance(peer_address()); - connection_socket_ = std::make_unique( - // Wraps the real IoHandle instance so that if this socket gets closed, - // the real IoHandle won't be affected. - std::make_unique(listener_config_.socket().ioHandle()), - std::move(local_addr), std::move(remote_addr)); - connection_socket_->setDetectedTransportProtocol( - Extensions::TransportSockets::TransportSocketNames::get().Quic); +EnvoyQuicConnection::~EnvoyQuicConnection() { connection_socket_->close(); } - filter_chain_ = listener_config_.filterChainManager().findFilterChain(*connection_socket_); - if (filter_chain_ == nullptr) { - listener_stats_.no_filter_chain_match_.inc(); - CloseConnection(quic::QUIC_CRYPTO_INTERNAL_ERROR, - "closing connection: no matching filter chain found for handshake", - quic::ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET); - return false; - } - const bool empty_filter_chain = !listener_config_.filterChainFactory().createNetworkFilterChain( - *envoy_connection_, filter_chain_->networkFilterFactories()); - if (empty_filter_chain) { - CloseConnection(quic::QUIC_CRYPTO_INTERNAL_ERROR, "closing connection: filter chain is empty", - quic::ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET); - return false; - } - } - return true; -} +uint64_t EnvoyQuicConnection::id() const { return envoy_connection_->id(); } } // namespace Quic } // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_connection.h b/source/extensions/quic_listeners/quiche/envoy_quic_connection.h index 04381bf383d1..dbc73e905749 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_connection.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_connection.h @@ -14,8 +14,6 @@ #include "common/common/logger.h" #include "envoy/network/connection.h" -#include "envoy/network/listener.h" -#include "server/connection_handler_impl.h" namespace Envoy { namespace Quic { @@ -28,43 +26,42 @@ class EnvoyQuicConnection : public quic::QuicConnection, EnvoyQuicConnection(const quic::QuicConnectionId& server_connection_id, quic::QuicSocketAddress initial_peer_address, quic::QuicConnectionHelperInterface& helper, - quic::QuicAlarmFactory& alarm_factory, quic::QuicPacketWriter& writer, + quic::QuicAlarmFactory& alarm_factory, quic::QuicPacketWriter* writer, bool owns_writer, quic::Perspective perspective, const quic::ParsedQuicVersionVector& supported_versions, - Network::ListenerConfig& listener_config, - Server::ListenerStats& listener_stats); + Network::ConnectionSocketPtr&& connection_socket); - ~EnvoyQuicConnection() override = default; + ~EnvoyQuicConnection() override; // Called by EnvoyQuicSession::setConnectionStats(). void setConnectionStats(const Network::Connection::ConnectionStats& stats) { connection_stats_ = std::make_unique(stats); } - // quic::QuicConnection - // Overridden to retrieve filter chain with initialized self address. - bool OnPacketHeader(const quic::QuicPacketHeader& header) override; - // Called in session Initialize(). void setEnvoyConnection(Network::Connection& connection) { envoy_connection_ = &connection; } const Network::ConnectionSocketPtr& connectionSocket() const { return connection_socket_; } + // Needed for ENVOY_CONN_LOG. + uint64_t id() const; + protected: Network::Connection::ConnectionStats& connectionStats() const { return *connection_stats_; } + Network::Connection& envoyConnection() const { + ASSERT(envoy_connection_ != nullptr); + return *envoy_connection_; + } + private: // TODO(danzh): populate stats. std::unique_ptr connection_stats_; - // Only initialized after self address is known. Must not own the underlying - // socket because UDP socket is shared among all connections. + // Assigned upon construction. Constructed with empty local address if unknown + // by then. Network::ConnectionSocketPtr connection_socket_; + // Points to an instance of EnvoyQuicServerSession or EnvoyQuicClientSession. Network::Connection* envoy_connection_{nullptr}; - Network::ListenerConfig& listener_config_; - Server::ListenerStats& listener_stats_; - // Latched to the corresponding quic FilterChain after connection_socket_ is - // initialized. - const Network::FilterChain* filter_chain_{nullptr}; }; } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc index e344eebc19ac..07126a290858 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_dispatcher.cc @@ -1,5 +1,6 @@ #include "extensions/quic_listeners/quiche/envoy_quic_dispatcher.h" +#include "extensions/quic_listeners/quiche/envoy_quic_server_connection.h" #include "extensions/quic_listeners/quiche/envoy_quic_server_session.h" namespace Envoy { @@ -37,10 +38,10 @@ void EnvoyQuicDispatcher::OnConnectionClosed(quic::QuicConnectionId connection_i quic::QuicSession* EnvoyQuicDispatcher::CreateQuicSession( quic::QuicConnectionId server_connection_id, const quic::QuicSocketAddress& peer_address, quic::QuicStringPiece /*alpn*/, const quic::ParsedQuicVersion& version) { - auto quic_connection = std::make_unique( - server_connection_id, peer_address, *helper(), *alarm_factory(), *writer(), - /*owns_writer=*/false, quic::Perspective::IS_SERVER, quic::ParsedQuicVersionVector{version}, - listener_config_, listener_stats_); + auto quic_connection = std::make_unique( + server_connection_id, peer_address, *helper(), *alarm_factory(), writer(), + /*owns_writer=*/false, quic::ParsedQuicVersionVector{version}, listener_config_, + listener_stats_); auto quic_session = new EnvoyQuicServerSession( config(), quic::ParsedQuicVersionVector{version}, std::move(quic_connection), this, session_helper(), crypto_config(), compressed_certs_cache(), dispatcher_); diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc index b173fa00c7b0..02360ae9b66c 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.cc @@ -1,5 +1,7 @@ #include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" +#include + #pragma GCC diagnostic push // QUICHE allows unused parameters. @@ -13,9 +15,12 @@ #include "extensions/quic_listeners/quiche/envoy_quic_utils.h" #include "common/buffer/buffer_impl.h" +#include "common/network/utility.h" namespace Envoy { namespace Quic { +EnvoyQuicPacketWriter::EnvoyQuicPacketWriter(Network::Socket& socket) + : write_blocked_(false), socket_(socket) {} quic::WriteResult EnvoyQuicPacketWriter::WritePacket(const char* buffer, size_t buf_len, const quic::QuicIpAddress& self_ip, @@ -24,17 +29,16 @@ quic::WriteResult EnvoyQuicPacketWriter::WritePacket(const char* buffer, size_t ASSERT(options == nullptr, "Per packet option is not supported yet."); ASSERT(!write_blocked_, "Cannot write while IO handle is blocked."); - Buffer::BufferFragmentImpl fragment(buffer, buf_len, nullptr); - Buffer::OwnedImpl buffer_wrapper; - buffer_wrapper.addBufferFragment(fragment); + Buffer::RawSlice slice; + slice.mem_ = const_cast(buffer); + slice.len_ = buf_len; quic::QuicSocketAddress self_address(self_ip, /*port=*/0); Network::Address::InstanceConstSharedPtr local_addr = quicAddressToEnvoyAddressInstance(self_address); Network::Address::InstanceConstSharedPtr remote_addr = quicAddressToEnvoyAddressInstance(peer_address); - Network::UdpSendData send_data{local_addr == nullptr ? nullptr : local_addr->ip(), *remote_addr, - buffer_wrapper}; - Api::IoCallUint64Result result = listener_.send(send_data); + Api::IoCallUint64Result result = Network::Utility::writeToSocket( + socket_, &slice, 1, local_addr == nullptr ? nullptr : local_addr->ip(), *remote_addr); if (result.ok()) { return {quic::WRITE_STATUS_OK, static_cast(result.rc_)}; } diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h index d82ac08ca7a7..55a6e5146d3a 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_packet_writer.h @@ -17,8 +17,7 @@ namespace Quic { class EnvoyQuicPacketWriter : public quic::QuicPacketWriter { public: - EnvoyQuicPacketWriter(Network::UdpListener& listener) - : write_blocked_(false), listener_(listener) {} + EnvoyQuicPacketWriter(Network::Socket& socket); quic::WriteResult WritePacket(const char* buffer, size_t buf_len, const quic::QuicIpAddress& self_address, @@ -44,7 +43,7 @@ class EnvoyQuicPacketWriter : public quic::QuicPacketWriter { private: // Modified by WritePacket() to indicate underlying IoHandle status. bool write_blocked_; - Network::UdpListener& listener_; + Network::Socket& socket_; }; } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_connection.cc b/source/extensions/quic_listeners/quiche/envoy_quic_server_connection.cc new file mode 100644 index 000000000000..52cbf66c2c82 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_connection.cc @@ -0,0 +1,61 @@ +#include "extensions/quic_listeners/quiche/envoy_quic_server_connection.h" + +#include "common/network/listen_socket_impl.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_utils.h" +#include "extensions/quic_listeners/quiche/quic_io_handle_wrapper.h" +#include "extensions/transport_sockets/well_known_names.h" + +namespace Envoy { +namespace Quic { + +EnvoyQuicServerConnection::EnvoyQuicServerConnection( + const quic::QuicConnectionId& server_connection_id, + quic::QuicSocketAddress initial_peer_address, quic::QuicConnectionHelperInterface& helper, + quic::QuicAlarmFactory& alarm_factory, quic::QuicPacketWriter* writer, bool owns_writer, + const quic::ParsedQuicVersionVector& supported_versions, + Network::ListenerConfig& listener_config, Server::ListenerStats& listener_stats) + : EnvoyQuicConnection( + server_connection_id, initial_peer_address, helper, alarm_factory, writer, owns_writer, + quic::Perspective::IS_SERVER, supported_versions, + std::make_unique( + // Wraps the real IoHandle instance so that if the connection socket gets closed, + // the real IoHandle won't be affected. + std::make_unique(listener_config.socket().ioHandle()), nullptr, + quicAddressToEnvoyAddressInstance(initial_peer_address))), + listener_config_(listener_config), listener_stats_(listener_stats) {} + +bool EnvoyQuicServerConnection::OnPacketHeader(const quic::QuicPacketHeader& header) { + if (!EnvoyQuicConnection::OnPacketHeader(header)) { + return false; + } + if (connectionSocket()->localAddress() != nullptr) { + return true; + } + ASSERT(self_address().IsInitialized()); + // Self address should be initialized by now. It's time to install filters. + connectionSocket()->setLocalAddress(quicAddressToEnvoyAddressInstance(self_address())); + connectionSocket()->setDetectedTransportProtocol( + Extensions::TransportSockets::TransportSocketNames::get().Quic); + ASSERT(filter_chain_ == nullptr); + filter_chain_ = listener_config_.filterChainManager().findFilterChain(*connectionSocket()); + if (filter_chain_ == nullptr) { + listener_stats_.no_filter_chain_match_.inc(); + CloseConnection(quic::QUIC_CRYPTO_INTERNAL_ERROR, + "closing connection: no matching filter chain found for handshake", + quic::ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET); + return false; + } + const bool empty_filter_chain = !listener_config_.filterChainFactory().createNetworkFilterChain( + envoyConnection(), filter_chain_->networkFilterFactories()); + if (empty_filter_chain) { + // TODO(danzh) check empty filter chain at config load time instead of here. + CloseConnection(quic::QUIC_CRYPTO_INTERNAL_ERROR, "closing connection: filter chain is empty", + quic::ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET); + return false; + } + return true; +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_connection.h b/source/extensions/quic_listeners/quiche/envoy_quic_server_connection.h new file mode 100644 index 000000000000..587e3a3e5fd6 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_connection.h @@ -0,0 +1,34 @@ +#include "envoy/network/listener.h" + +#include "server/connection_handler_impl.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_connection.h" + +namespace Envoy { +namespace Quic { + +class EnvoyQuicServerConnection : public EnvoyQuicConnection { +public: + EnvoyQuicServerConnection(const quic::QuicConnectionId& server_connection_id, + quic::QuicSocketAddress initial_peer_address, + quic::QuicConnectionHelperInterface& helper, + quic::QuicAlarmFactory& alarm_factory, quic::QuicPacketWriter* writer, + bool owns_writer, + const quic::ParsedQuicVersionVector& supported_versions, + Network::ListenerConfig& listener_config, + Server::ListenerStats& listener_stats); + + // EnvoyQuicConnection + // Overridden to set connection_socket_ with initialized self address and retrieve filter chain. + bool OnPacketHeader(const quic::QuicPacketHeader& header) override; + +private: + Network::ListenerConfig& listener_config_; + Server::ListenerStats& listener_stats_; + // Latched to the corresponding quic FilterChain after connection_socket_ is + // initialized. + const Network::FilterChain* filter_chain_{nullptr}; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_session.cc b/source/extensions/quic_listeners/quiche/envoy_quic_server_session.cc index a6452fca1b12..021c390e474d 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_server_session.cc +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_session.cc @@ -21,10 +21,10 @@ EnvoyQuicServerSession::EnvoyQuicServerSession( quic::QuicCompressedCertsCache* compressed_certs_cache, Event::Dispatcher& dispatcher) : quic::QuicServerSessionBase(config, supported_versions, connection.get(), visitor, helper, crypto_config, compressed_certs_cache), - quic_connection_(std::move(connection)), filter_manager_(*this), dispatcher_(dispatcher), - stream_info_(dispatcher.timeSource()) { - // TODO(danzh): Use QUIC specific enum value. - stream_info_.protocol(Http::Protocol::Http2); + QuicFilterManagerConnectionImpl(std::move(connection), dispatcher) {} + +absl::string_view EnvoyQuicServerSession::requestedServerName() const { + return {GetCryptoStream()->crypto_negotiated_params().sni}; } quic::QuicCryptoServerStreamBase* EnvoyQuicServerSession::CreateQuicCryptoServerStream( @@ -68,14 +68,7 @@ void EnvoyQuicServerSession::setUpRequestDecoder(EnvoyQuicStream& stream) { void EnvoyQuicServerSession::OnConnectionClosed(const quic::QuicConnectionCloseFrame& frame, quic::ConnectionCloseSource source) { quic::QuicServerSessionBase::OnConnectionClosed(frame, source); - for (auto callback : network_connection_callbacks_) { - // Tell filters about connection close. - callback->onEvent(source == quic::ConnectionCloseSource::FROM_PEER - ? Network::ConnectionEvent::RemoteClose - : Network::ConnectionEvent::LocalClose); - } - transport_failure_reason_ = absl::StrCat(quic::QuicErrorCodeToString(frame.quic_error_code), - " with details: ", frame.error_details); + onConnectionCloseEvent(frame, source); } void EnvoyQuicServerSession::Initialize() { @@ -89,101 +82,11 @@ void EnvoyQuicServerSession::SendGoAway(quic::QuicErrorCode error_code, const st } } -void EnvoyQuicServerSession::addWriteFilter(Network::WriteFilterSharedPtr filter) { - filter_manager_.addWriteFilter(filter); -} - -void EnvoyQuicServerSession::addFilter(Network::FilterSharedPtr filter) { - filter_manager_.addFilter(filter); -} - -void EnvoyQuicServerSession::addReadFilter(Network::ReadFilterSharedPtr filter) { - filter_manager_.addReadFilter(filter); -} - -bool EnvoyQuicServerSession::initializeReadFilters() { - return filter_manager_.initializeReadFilters(); -} - -void EnvoyQuicServerSession::addConnectionCallbacks(Network::ConnectionCallbacks& cb) { - network_connection_callbacks_.push_back(&cb); -} - -void EnvoyQuicServerSession::addBytesSentCallback(Network::Connection::BytesSentCb /*cb*/) { - // TODO(danzh): implement to support proxy. This interface is only called from - // TCP proxy code. - ASSERT(false, "addBytesSentCallback is not implemented for QUIC"); -} - -void EnvoyQuicServerSession::enableHalfClose(bool enabled) { - ASSERT(!enabled, "Quic connection doesn't support half close."); -} - -void EnvoyQuicServerSession::setBufferLimits(uint32_t /*limit*/) { - // TODO(danzh): add interface to quic for connection level buffer throttling. - // Currently read buffer is capped by connection level flow control. And - // write buffer is not capped. - ENVOY_CONN_LOG(error, "Quic manages its own buffer currently.", *this); -} - -uint32_t EnvoyQuicServerSession::bufferLimit() const { - // As quic connection is not HTTP1.1, this method shouldn't be called by HCM. - NOT_REACHED_GCOVR_EXCL_LINE; -} - -void EnvoyQuicServerSession::close(Network::ConnectionCloseType type) { - if (type != Network::ConnectionCloseType::NoFlush) { - // TODO(danzh): Implement FlushWrite and FlushWriteAndDelay mode. - ENVOY_CONN_LOG(error, "Flush write is not implemented for QUIC.", *this); +void EnvoyQuicServerSession::OnCryptoHandshakeEvent(CryptoHandshakeEvent event) { + quic::QuicServerSessionBase::OnCryptoHandshakeEvent(event); + if (event == HANDSHAKE_CONFIRMED) { + raiseEvent(Network::ConnectionEvent::Connected); } - connection()->CloseConnection(quic::QUIC_NO_ERROR, "Closed by application", - quic::ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET); -} - -void EnvoyQuicServerSession::setDelayedCloseTimeout(std::chrono::milliseconds timeout) { - ASSERT(timeout == std::chrono::milliseconds::zero(), - "Delayed close of connection is not supported"); -} - -std::chrono::milliseconds EnvoyQuicServerSession::delayedCloseTimeout() const { - // Not called outside of Network::ConnectionImpl. Maybe remove this interface - // from Network::Connection. - NOT_REACHED_GCOVR_EXCL_LINE; -} - -const Network::ConnectionSocket::OptionsSharedPtr& EnvoyQuicServerSession::socketOptions() const { - ENVOY_CONN_LOG( - error, - "QUIC connection socket is merely a wrapper, and doesn't have any specific socket options.", - *this); - return quic_connection_->connectionSocket()->options(); -} - -absl::string_view EnvoyQuicServerSession::requestedServerName() const { - return {GetCryptoStream()->crypto_negotiated_params().sni}; -} - -const Network::Address::InstanceConstSharedPtr& EnvoyQuicServerSession::remoteAddress() const { - ASSERT(quic_connection_->connectionSocket() != nullptr, - "remoteAddress() should only be called after OnPacketHeader"); - return quic_connection_->connectionSocket()->remoteAddress(); -} - -const Network::Address::InstanceConstSharedPtr& EnvoyQuicServerSession::localAddress() const { - ASSERT(quic_connection_->connectionSocket() != nullptr, - "localAddress() should only be called after OnPacketHeader"); - return quic_connection_->connectionSocket()->localAddress(); -} - -Ssl::ConnectionInfoConstSharedPtr EnvoyQuicServerSession::ssl() const { - // TODO(danzh): construct Ssl::ConnectionInfo from crypto stream - ENVOY_CONN_LOG(error, "Ssl::ConnectionInfo instance is not populated.", *this); - return nullptr; -} - -void EnvoyQuicServerSession::rawWrite(Buffer::Instance& /*data*/, bool /*end_stream*/) { - // Network filter should stop iteration. - NOT_REACHED_GCOVR_EXCL_LINE; } } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/envoy_quic_server_session.h b/source/extensions/quic_listeners/quiche/envoy_quic_server_session.h index 0cb2d0e84d4e..d849aa0161ca 100644 --- a/source/extensions/quic_listeners/quiche/envoy_quic_server_session.h +++ b/source/extensions/quic_listeners/quiche/envoy_quic_server_session.h @@ -13,22 +13,15 @@ #include -#include "common/common/logger.h" -#include "common/common/empty_string.h" -#include "common/network/filter_manager_impl.h" -#include "common/stream_info/stream_info_impl.h" -#include "envoy/network/connection.h" -#include "envoy/event/dispatcher.h" +#include "extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h" #include "extensions/quic_listeners/quiche/envoy_quic_stream.h" -#include "extensions/quic_listeners/quiche/envoy_quic_connection.h" namespace Envoy { namespace Quic { // Act as a Network::Connection to HCM and a FilterManager to FilterFactoryCb. class EnvoyQuicServerSession : public quic::QuicServerSessionBase, - public Network::FilterManagerConnection, - protected Logger::Loggable { + public QuicFilterManagerConnectionImpl { public: EnvoyQuicServerSession(const quic::QuicConfig& config, const quic::ParsedQuicVersionVector& supported_versions, @@ -39,81 +32,8 @@ class EnvoyQuicServerSession : public quic::QuicServerSessionBase, quic::QuicCompressedCertsCache* compressed_certs_cache, Event::Dispatcher& dispatcher); - // Network::FilterManager - // Overridden to delegate calls to filter_manager_. - void addWriteFilter(Network::WriteFilterSharedPtr filter) override; - void addFilter(Network::FilterSharedPtr filter) override; - void addReadFilter(Network::ReadFilterSharedPtr filter) override; - bool initializeReadFilters() override; - // Network::Connection - void addConnectionCallbacks(Network::ConnectionCallbacks& cb) override; - void addBytesSentCallback(Network::Connection::BytesSentCb /*cb*/) override; - void enableHalfClose(bool enabled) override; - void close(Network::ConnectionCloseType type) override; - Event::Dispatcher& dispatcher() override { return dispatcher_; } - uint64_t id() const override { - // QUIC connection id can be 18 types. It's easier to use hash value instead - // of trying to map it into a 64-bit space. - return connection_id().Hash(); - } - std::string nextProtocol() const override { return EMPTY_STRING; } - void noDelay(bool /*enable*/) override { - // No-op. TCP_NODELAY doesn't apply to UDP. - } - void setDelayedCloseTimeout(std::chrono::milliseconds timeout) override; - std::chrono::milliseconds delayedCloseTimeout() const override; - void readDisable(bool disable) override { - ASSERT(!disable, "Quic connection should be able to read through out its life time."); - } - void detectEarlyCloseWhenReadDisabled(bool /*value*/) override { NOT_REACHED_GCOVR_EXCL_LINE; } - bool readEnabled() const override { return true; } - const Network::Address::InstanceConstSharedPtr& remoteAddress() const override; - const Network::Address::InstanceConstSharedPtr& localAddress() const override; - absl::optional - unixSocketPeerCredentials() const override { - ASSERT(false, "Unix domain socket is not supported."); - return absl::nullopt; - } - void setConnectionStats(const Network::Connection::ConnectionStats& stats) override { - stats_ = std::make_unique(stats); - quic_connection_->setConnectionStats(stats); - } - Ssl::ConnectionInfoConstSharedPtr ssl() const override; - Network::Connection::State state() const override { - return connection()->connected() ? Network::Connection::State::Open - : Network::Connection::State::Closed; - } - void write(Buffer::Instance& /*data*/, bool /*end_stream*/) override { - // All writes should be handled by Quic internally. - NOT_REACHED_GCOVR_EXCL_LINE; - } - void setBufferLimits(uint32_t limit) override; - uint32_t bufferLimit() const override; - bool localAddressRestored() const override { - // SO_ORIGINAL_DST not supported by QUIC. - return false; - } - bool aboveHighWatermark() const override { - ENVOY_CONN_LOG(error, "QUIC doesn't have connection level write buffer limit.", *this); - return false; - } - const Network::ConnectionSocket::OptionsSharedPtr& socketOptions() const override; absl::string_view requestedServerName() const override; - StreamInfo::StreamInfo& streamInfo() override { return stream_info_; } - const StreamInfo::StreamInfo& streamInfo() const override { return stream_info_; } - absl::string_view transportFailureReason() const override { return transport_failure_reason_; } - - // Network::FilterManagerConnection - void rawWrite(Buffer::Instance& data, bool end_stream) override; - - // Network::ReadBufferSource - Network::StreamBuffer getReadBuffer() override { - // Network filter has to stop iteration to prevent hitting this line. - NOT_REACHED_GCOVR_EXCL_LINE; - } - // Network::WriteBufferSource - Network::StreamBuffer getWriteBuffer() override { NOT_REACHED_GCOVR_EXCL_LINE; } // Called by QuicHttpServerConnectionImpl before creating data streams. void setHttpConnectionCallbacks(Http::ServerConnectionCallbacks& callbacks) { @@ -125,6 +45,8 @@ class EnvoyQuicServerSession : public quic::QuicServerSessionBase, quic::ConnectionCloseSource source) override; void Initialize() override; void SendGoAway(quic::QuicErrorCode error_code, const std::string& reason) override; + // quic::QuicSpdySession + void OnCryptoHandshakeEvent(CryptoHandshakeEvent event) override; protected: // quic::QuicServerSessionBase @@ -142,21 +64,9 @@ class EnvoyQuicServerSession : public quic::QuicServerSessionBase, private: void setUpRequestDecoder(EnvoyQuicStream& stream); - std::unique_ptr quic_connection_; - // Currently ConnectionManagerImpl is the one and only filter. If more network - // filters are added, ConnectionManagerImpl should always be the last one. - // Its onRead() is only called once to trigger ReadFilter::onNewConnection() - // and the rest incoming data bypasses these filters. - Network::FilterManagerImpl filter_manager_; - Event::Dispatcher& dispatcher_; - StreamInfo::StreamInfoImpl stream_info_; - std::string transport_failure_reason_; - // TODO(danzh): populate stats. - std::unique_ptr stats_; // These callbacks are owned by network filters and quic session should out live // them. Http::ServerConnectionCallbacks* http_connection_callbacks_{nullptr}; - std::list network_connection_callbacks_; }; } // namespace Quic diff --git a/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.cc b/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.cc new file mode 100644 index 000000000000..edfe62051d3c --- /dev/null +++ b/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.cc @@ -0,0 +1,116 @@ +#include "extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h" + +#include + +namespace Envoy { +namespace Quic { + +QuicFilterManagerConnectionImpl::QuicFilterManagerConnectionImpl( + std::unique_ptr connection, Event::Dispatcher& dispatcher) + : quic_connection_(std::move(connection)), filter_manager_(*this), dispatcher_(dispatcher), + // QUIC connection id can be 18 bytes. It's easier to use hash value instead + // of trying to map it into a 64-bit space. + stream_info_(dispatcher.timeSource()), id_(quic_connection_->connection_id().Hash()) { + // TODO(danzh): Use QUIC specific enum value. + stream_info_.protocol(Http::Protocol::Http2); +} + +void QuicFilterManagerConnectionImpl::addWriteFilter(Network::WriteFilterSharedPtr filter) { + filter_manager_.addWriteFilter(filter); +} + +void QuicFilterManagerConnectionImpl::addFilter(Network::FilterSharedPtr filter) { + filter_manager_.addFilter(filter); +} + +void QuicFilterManagerConnectionImpl::addReadFilter(Network::ReadFilterSharedPtr filter) { + filter_manager_.addReadFilter(filter); +} + +bool QuicFilterManagerConnectionImpl::initializeReadFilters() { + return filter_manager_.initializeReadFilters(); +} + +void QuicFilterManagerConnectionImpl::addConnectionCallbacks(Network::ConnectionCallbacks& cb) { + network_connection_callbacks_.push_back(&cb); +} + +void QuicFilterManagerConnectionImpl::enableHalfClose(bool enabled) { + RELEASE_ASSERT(!enabled, "Quic connection doesn't support half close."); +} + +void QuicFilterManagerConnectionImpl::setBufferLimits(uint32_t /*limit*/) { + // TODO(danzh): add interface to quic for connection level buffer throttling. + // Currently read buffer is capped by connection level flow control. And + // write buffer is not capped. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void QuicFilterManagerConnectionImpl::close(Network::ConnectionCloseType type) { + if (type != Network::ConnectionCloseType::NoFlush) { + // TODO(danzh): Implement FlushWrite and FlushWriteAndDelay mode. + } + quic_connection_->CloseConnection(quic::QUIC_NO_ERROR, "Closed by application", + quic::ConnectionCloseBehavior::SEND_CONNECTION_CLOSE_PACKET); +} + +void QuicFilterManagerConnectionImpl::setDelayedCloseTimeout(std::chrono::milliseconds timeout) { + if (timeout != std::chrono::milliseconds::zero()) { + // TODO(danzh) support delayed close of connection. + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } +} + +std::chrono::milliseconds QuicFilterManagerConnectionImpl::delayedCloseTimeout() const { + // Not called outside of Network::ConnectionImpl. + // TODO(#8419): Try remove this interface from Network::Connection. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +const Network::ConnectionSocket::OptionsSharedPtr& +QuicFilterManagerConnectionImpl::socketOptions() const { + return quic_connection_->connectionSocket()->options(); +} + +const Network::Address::InstanceConstSharedPtr& +QuicFilterManagerConnectionImpl::remoteAddress() const { + ASSERT(quic_connection_->connectionSocket() != nullptr, + "remoteAddress() should only be called after OnPacketHeader"); + return quic_connection_->connectionSocket()->remoteAddress(); +} + +const Network::Address::InstanceConstSharedPtr& +QuicFilterManagerConnectionImpl::localAddress() const { + ASSERT(quic_connection_->connectionSocket() != nullptr, + "localAddress() should only be called after OnPacketHeader"); + return quic_connection_->connectionSocket()->localAddress(); +} + +Ssl::ConnectionInfoConstSharedPtr QuicFilterManagerConnectionImpl::ssl() const { + // TODO(danzh): construct Ssl::ConnectionInfo from crypto stream + return nullptr; +} + +void QuicFilterManagerConnectionImpl::rawWrite(Buffer::Instance& /*data*/, bool /*end_stream*/) { + // Network filter should stop iteration. + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void QuicFilterManagerConnectionImpl::onConnectionCloseEvent( + const quic::QuicConnectionCloseFrame& frame, quic::ConnectionCloseSource source) { + // Tell network callbacks about connection close. + raiseEvent(source == quic::ConnectionCloseSource::FROM_PEER + ? Network::ConnectionEvent::RemoteClose + : Network::ConnectionEvent::LocalClose); + transport_failure_reason_ = absl::StrCat(quic::QuicErrorCodeToString(frame.quic_error_code), + " with details: ", frame.error_details); +} + +void QuicFilterManagerConnectionImpl::raiseEvent(Network::ConnectionEvent event) { + for (auto callback : network_connection_callbacks_) { + callback->onEvent(event); + } +} + +} // namespace Quic +} // namespace Envoy diff --git a/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h b/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h new file mode 100644 index 000000000000..fc8f57df6a03 --- /dev/null +++ b/source/extensions/quic_listeners/quiche/quic_filter_manager_connection_impl.h @@ -0,0 +1,127 @@ +#pragma once + +#include "envoy/event/dispatcher.h" +#include "envoy/network/connection.h" + +#include "common/common/empty_string.h" +#include "common/common/logger.h" +#include "common/network/filter_manager_impl.h" +#include "common/stream_info/stream_info_impl.h" + +#include "extensions/quic_listeners/quiche/envoy_quic_connection.h" + +namespace Envoy { +namespace Quic { + +// Act as a Network::Connection to HCM and a FilterManager to FilterFactoryCb. +class QuicFilterManagerConnectionImpl : public Network::FilterManagerConnection, + protected Logger::Loggable { +public: + QuicFilterManagerConnectionImpl(std::unique_ptr connection, + Event::Dispatcher& dispatcher); + + // Network::FilterManager + // Overridden to delegate calls to filter_manager_. + void addWriteFilter(Network::WriteFilterSharedPtr filter) override; + void addFilter(Network::FilterSharedPtr filter) override; + void addReadFilter(Network::ReadFilterSharedPtr filter) override; + bool initializeReadFilters() override; + + // Network::Connection + void addConnectionCallbacks(Network::ConnectionCallbacks& cb) override; + void addBytesSentCallback(Network::Connection::BytesSentCb /*cb*/) override { + // TODO(danzh): implement to support proxy. This interface is only called from + // TCP proxy code. + NOT_REACHED_GCOVR_EXCL_LINE; + } + void enableHalfClose(bool enabled) override; + void close(Network::ConnectionCloseType type) override; + Event::Dispatcher& dispatcher() override { return dispatcher_; } + uint64_t id() const override { return id_; } + std::string nextProtocol() const override { return EMPTY_STRING; } + void noDelay(bool /*enable*/) override { + // No-op. TCP_NODELAY doesn't apply to UDP. + } + void setDelayedCloseTimeout(std::chrono::milliseconds timeout) override; + std::chrono::milliseconds delayedCloseTimeout() const override; + void readDisable(bool /*disable*/) override { NOT_REACHED_GCOVR_EXCL_LINE; } + void detectEarlyCloseWhenReadDisabled(bool /*value*/) override { NOT_REACHED_GCOVR_EXCL_LINE; } + bool readEnabled() const override { return true; } + const Network::Address::InstanceConstSharedPtr& remoteAddress() const override; + const Network::Address::InstanceConstSharedPtr& localAddress() const override; + absl::optional + unixSocketPeerCredentials() const override { + // Unix domain socket is not supported. + NOT_REACHED_GCOVR_EXCL_LINE; + } + void setConnectionStats(const Network::Connection::ConnectionStats& stats) override { + stats_ = std::make_unique(stats); + quic_connection_->setConnectionStats(stats); + } + Ssl::ConnectionInfoConstSharedPtr ssl() const override; + Network::Connection::State state() const override { + return quic_connection_->connected() ? Network::Connection::State::Open + : Network::Connection::State::Closed; + } + void write(Buffer::Instance& /*data*/, bool /*end_stream*/) override { + // All writes should be handled by Quic internally. + NOT_REACHED_GCOVR_EXCL_LINE; + } + void setBufferLimits(uint32_t limit) override; + uint32_t bufferLimit() const override { + // As quic connection is not HTTP1.1, this method shouldn't be called by HCM. + NOT_REACHED_GCOVR_EXCL_LINE; + } + bool localAddressRestored() const override { + // SO_ORIGINAL_DST not supported by QUIC. + NOT_REACHED_GCOVR_EXCL_LINE; + } + bool aboveHighWatermark() const override { + // TODO(danzh) Aggregate the write buffer usage cross all the streams and + // add an upper limit for this connection. + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + const Network::ConnectionSocket::OptionsSharedPtr& socketOptions() const override; + StreamInfo::StreamInfo& streamInfo() override { return stream_info_; } + const StreamInfo::StreamInfo& streamInfo() const override { return stream_info_; } + absl::string_view transportFailureReason() const override { return transport_failure_reason_; } + + // Network::FilterManagerConnection + void rawWrite(Buffer::Instance& data, bool end_stream) override; + + // Network::ReadBufferSource + Network::StreamBuffer getReadBuffer() override { + // Network filter has to stop iteration to prevent hitting this line. + NOT_REACHED_GCOVR_EXCL_LINE; + } + // Network::WriteBufferSource + Network::StreamBuffer getWriteBuffer() override { NOT_REACHED_GCOVR_EXCL_LINE; } + +protected: + // Propagate connection close to network_connection_callbacks_. + void onConnectionCloseEvent(const quic::QuicConnectionCloseFrame& frame, + quic::ConnectionCloseSource source); + + void raiseEvent(Network::ConnectionEvent event); + + std::unique_ptr quic_connection_; + // TODO(danzh): populate stats. + std::unique_ptr stats_; + +private: + // Currently ConnectionManagerImpl is the one and only filter. If more network + // filters are added, ConnectionManagerImpl should always be the last one. + // Its onRead() is only called once to trigger ReadFilter::onNewConnection() + // and the rest incoming data bypasses these filters. + Network::FilterManagerImpl filter_manager_; + Event::Dispatcher& dispatcher_; + StreamInfo::StreamInfoImpl stream_info_; + // These callbacks are owned by network filters and quic session should out live + // them. + std::list network_connection_callbacks_; + std::string transport_failure_reason_; + const uint64_t id_; +}; + +} // namespace Quic +} // namespace Envoy diff --git a/test/extensions/quic_listeners/quiche/BUILD b/test/extensions/quic_listeners/quiche/BUILD index b84e7ba9c86a..74659b582958 100644 --- a/test/extensions/quic_listeners/quiche/BUILD +++ b/test/extensions/quic_listeners/quiche/BUILD @@ -34,7 +34,9 @@ envoy_cc_test( deps = [ "//source/common/network:io_socket_error_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_packet_writer_lib", + "//test/mocks/api:api_mocks", "//test/mocks/network:network_mocks", + "//test/test_common:threadsafe_singleton_injector_lib", ], ) @@ -59,7 +61,7 @@ envoy_cc_test( "//source/common/http:headers_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_factory_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_connection_helper_lib", - "//source/extensions/quic_listeners/quiche:envoy_quic_connection_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_server_connection_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_server_stream_lib", "//test/mocks/http:stream_decoder_mock", "//test/mocks/network:network_mocks", @@ -75,10 +77,11 @@ envoy_cc_test( deps = [ ":quic_test_utils_for_envoy_lib", "//include/envoy/stats:stats_macros", - "//source/extensions/quic_listeners/quiche:codec_impl_lib", + "//source/extensions/quic_listeners/quiche:codec_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_alarm_factory_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_connection_helper_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_proof_source_lib", + "//source/extensions/quic_listeners/quiche:envoy_quic_server_connection_lib", "//source/extensions/quic_listeners/quiche:envoy_quic_server_session_lib", "//source/server:configuration_lib", "//test/mocks/event:event_mocks", diff --git a/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc b/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc index 03e32d788b62..c04c04b512cd 100644 --- a/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc +++ b/test/extensions/quic_listeners/quiche/active_quic_listener_test.cc @@ -158,6 +158,7 @@ TEST_P(ActiveQuicListenerTest, ReceiveFullQuicCHLO) { dispatcher_->exit(); return Network::FilterStatus::StopIteration; })); + EXPECT_CALL(network_connection_callbacks_, onEvent(Network::ConnectionEvent::Connected)); dispatcher_->run(Event::Dispatcher::RunType::Block); diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc index b42ab88d606f..2d1c44a4ab4f 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_dispatcher_test.cc @@ -181,6 +181,7 @@ TEST_P(EnvoyQuicDispatcherTest, CreateNewConnectionUponCHLO) { EXPECT_CALL(*read_filter, onNewConnection()) // Stop iteration to avoid calling getRead/WriteBuffer(). .WillOnce(Invoke([]() { return Network::FilterStatus::StopIteration; })); + EXPECT_CALL(network_connection_callbacks, onEvent(Network::ConnectionEvent::Connected)); quic::QuicConnectionId connection_id = quic::test::TestConnectionId(1); // Upon receiving a full CHLO. A new quic connection should be created and have its filter diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc index 8b5f127cbcf1..013c98851be4 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_server_session_test.cc @@ -13,6 +13,7 @@ #include #include "extensions/quic_listeners/quiche/envoy_quic_server_session.h" +#include "extensions/quic_listeners/quiche/envoy_quic_server_connection.h" #include "extensions/quic_listeners/quiche/codec_impl.h" #include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" #include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" @@ -44,17 +45,18 @@ using testing::ReturnRef; namespace Envoy { namespace Quic { -class TestEnvoyQuicConnection : public EnvoyQuicConnection { +class TestEnvoyQuicServerConnection : public EnvoyQuicServerConnection { public: - TestEnvoyQuicConnection(quic::QuicConnectionHelperInterface& helper, - quic::QuicAlarmFactory& alarm_factory, quic::QuicPacketWriter& writer, - const quic::ParsedQuicVersionVector& supported_versions, - Network::ListenerConfig& listener_config, Server::ListenerStats& stats) - : EnvoyQuicConnection(quic::test::TestConnectionId(), - quic::QuicSocketAddress(quic::QuicIpAddress::Loopback4(), 12345), - helper, alarm_factory, writer, /*owns_writer=*/false, - quic::Perspective::IS_SERVER, supported_versions, listener_config, - stats) {} + TestEnvoyQuicServerConnection(quic::QuicConnectionHelperInterface& helper, + quic::QuicAlarmFactory& alarm_factory, + quic::QuicPacketWriter& writer, + const quic::ParsedQuicVersionVector& supported_versions, + Network::ListenerConfig& listener_config, + Server::ListenerStats& stats) + : EnvoyQuicServerConnection(quic::test::TestConnectionId(), + quic::QuicSocketAddress(quic::QuicIpAddress::Loopback4(), 12345), + helper, alarm_factory, &writer, /*owns_writer=*/false, + supported_versions, listener_config, stats) {} Network::Connection::ConnectionStats& connectionStats() const { return EnvoyQuicConnection::connectionStats(); @@ -76,14 +78,14 @@ class EnvoyQuicServerSessionTest : public testing::TestWithParam { listener_stats_({ALL_LISTENER_STATS(POOL_COUNTER(listener_config_.listenerScope()), POOL_GAUGE(listener_config_.listenerScope()), POOL_HISTOGRAM(listener_config_.listenerScope()))}), - quic_connection_(new TestEnvoyQuicConnection(connection_helper_, alarm_factory_, writer_, - quic_version_, listener_config_, - listener_stats_)), + quic_connection_(new TestEnvoyQuicServerConnection(connection_helper_, alarm_factory_, + writer_, quic_version_, listener_config_, + listener_stats_)), crypto_config_(quic::QuicCryptoServerConfig::TESTING, quic::QuicRandom::GetInstance(), std::make_unique(), quic::KeyExchangeSource::Default()), envoy_quic_session_(quic_config_, quic_version_, - std::unique_ptr(quic_connection_), + std::unique_ptr(quic_connection_), /*visitor=*/nullptr, &crypto_stream_helper_, &crypto_config_, &compressed_certs_cache_, *dispatcher_), read_filter_(new Network::MockReadFilter()) { @@ -138,7 +140,7 @@ class EnvoyQuicServerSessionTest : public testing::TestWithParam { testing::NiceMock writer_; testing::NiceMock listener_config_; Server::ListenerStats listener_stats_; - TestEnvoyQuicConnection* quic_connection_; + TestEnvoyQuicServerConnection* quic_connection_; quic::QuicConfig quic_config_; quic::QuicCryptoServerConfig crypto_config_; testing::NiceMock crypto_stream_helper_; @@ -304,18 +306,16 @@ TEST_P(EnvoyQuicServerSessionTest, FlushCloseNotSupported) { EXPECT_CALL(*quic_connection_, SendConnectionClosePacket(quic::QUIC_NO_ERROR, "Closed by application")); EXPECT_CALL(network_connection_callbacks_, onEvent(Network::ConnectionEvent::LocalClose)); - EXPECT_LOG_CONTAINS("error", "Flush write is not implemented for QUIC.", - envoy_quic_session_.close(Network::ConnectionCloseType::FlushWrite)); + envoy_quic_session_.close(Network::ConnectionCloseType::FlushWrite); } TEST_P(EnvoyQuicServerSessionTest, ShutdownNotice) { installReadFilter(); // Not verifying dummy implementation, just to have coverage. - EXPECT_DEBUG_DEATH(envoy_quic_session_.enableHalfClose(true), ""); - envoy_quic_session_.setBufferLimits(1024 * 1024); + EXPECT_DEATH(envoy_quic_session_.enableHalfClose(true), ""); EXPECT_EQ(nullptr, envoy_quic_session_.ssl()); - EXPECT_FALSE(envoy_quic_session_.aboveHighWatermark()); - EXPECT_DEBUG_DEATH(envoy_quic_session_.setDelayedCloseTimeout(std::chrono::milliseconds(1)), ""); + EXPECT_DEATH(envoy_quic_session_.aboveHighWatermark(), ""); + EXPECT_DEATH(envoy_quic_session_.setDelayedCloseTimeout(std::chrono::milliseconds(1)), ""); http_connection_->shutdownNotice(); } @@ -390,11 +390,6 @@ TEST_P(EnvoyQuicServerSessionTest, NetworkConnectionInterface) { installReadFilter(); EXPECT_EQ(dispatcher_.get(), &envoy_quic_session_.dispatcher()); EXPECT_TRUE(envoy_quic_session_.readEnabled()); - EXPECT_FALSE(envoy_quic_session_.localAddressRestored()); - EXPECT_DEBUG_DEATH(envoy_quic_session_.unixSocketPeerCredentials(), - "Unix domain socket is not supported."); - EXPECT_DEBUG_DEATH(envoy_quic_session_.readDisable(true), - "Quic connection should be able to read through out its life time."); } } // namespace Quic diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc index 31df75c0a2b0..b77714985cc1 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_server_stream_test.cc @@ -18,7 +18,7 @@ #include "common/http/headers.h" #include "test/test_common/utility.h" #include "extensions/quic_listeners/quiche/envoy_quic_alarm_factory.h" -#include "extensions/quic_listeners/quiche/envoy_quic_connection.h" +#include "extensions/quic_listeners/quiche/envoy_quic_server_connection.h" #include "extensions/quic_listeners/quiche/envoy_quic_connection_helper.h" #include "test/mocks/http/stream_decoder.h" #include "test/mocks/network/mocks.h" @@ -68,9 +68,8 @@ class EnvoyQuicServerStreamTest : public testing::TestWithParam { POOL_HISTOGRAM(listener_config_.listenerScope()))}), quic_connection_(quic::test::TestConnectionId(), quic::QuicSocketAddress(quic::QuicIpAddress::Any6(), 12345), - connection_helper_, alarm_factory_, writer_, - /*owns_writer=*/false, quic::Perspective::IS_SERVER, {quic_version_}, - listener_config_, listener_stats_), + connection_helper_, alarm_factory_, &writer_, + /*owns_writer=*/false, {quic_version_}, listener_config_, listener_stats_), quic_session_(quic_config_, {quic_version_}, &quic_connection_, /*visitor=*/nullptr, /*helper=*/nullptr, /*crypto_config=*/nullptr, /*compressed_certs_cache=*/nullptr), @@ -107,7 +106,7 @@ class EnvoyQuicServerStreamTest : public testing::TestWithParam { quic::QuicConfig quic_config_; testing::NiceMock listener_config_; Server::ListenerStats listener_stats_; - EnvoyQuicConnection quic_connection_; + EnvoyQuicServerConnection quic_connection_; MockQuicServerSession quic_session_; quic::QuicStreamId stream_id_; EnvoyQuicServerStream quic_stream_; diff --git a/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc b/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc index f8ef123bd736..d418531ba359 100644 --- a/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc +++ b/test/extensions/quic_listeners/quiche/envoy_quic_writer_test.cc @@ -1,42 +1,54 @@ +#include + #include #include +#include "common/network/address_impl.h" #include "common/network/io_socket_error_impl.h" #include "extensions/quic_listeners/quiche/envoy_quic_packet_writer.h" +#include "test/mocks/api/mocks.h" #include "test/mocks/network/mocks.h" +#include "test/test_common/threadsafe_singleton_injector.h" #include "gmock/gmock.h" #include "gtest/gtest.h" +using testing::Return; + namespace Envoy { namespace Quic { class EnvoyQuicWriterTest : public ::testing::Test { public: - EnvoyQuicWriterTest() : envoy_quic_writer_(udp_listener_) { - self_address_.FromString("0.0.0.0"); + EnvoyQuicWriterTest() : envoy_quic_writer_(socket_) { + self_address_.FromString("::"); quic::QuicIpAddress peer_ip; - peer_ip.FromString("127.0.0.1"); + peer_ip.FromString("::1"); peer_address_ = quic::QuicSocketAddress(peer_ip, /*port=*/123); - EXPECT_CALL(udp_listener_, onDestroy()); - ON_CALL(udp_listener_, send(_)) - .WillByDefault(testing::Invoke([](const Network::UdpSendData& send_data) { - return Api::IoCallUint64Result( - send_data.buffer_.length(), - Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)); - })); + ON_CALL(os_sys_calls_, socket(_, _, _)).WillByDefault(Return(Api::SysCallIntResult{3, 0})); + ON_CALL(os_sys_calls_, close(3)).WillByDefault(Return(Api::SysCallIntResult{0, 0})); } - void verifySendData(const std::string& content, const Network::UdpSendData send_data) { - EXPECT_EQ(peer_address_.ToString(), send_data.peer_address_.asString()); - EXPECT_EQ(self_address_.ToString(), send_data.local_ip_->addressAsString()); - EXPECT_EQ(content, send_data.buffer_.toString()); + void verifySendData(const std::string& content, const msghdr* message) { + EXPECT_EQ(peer_address_.ToString(), Network::Address::addressFromSockAddr( + *reinterpret_cast(message->msg_name), + message->msg_namelen, /*v6only=*/false) + ->asString()); + cmsghdr* const cmsg = CMSG_FIRSTHDR(message); + auto pktinfo = reinterpret_cast(CMSG_DATA(cmsg)); + EXPECT_EQ(0, memcmp(self_address_.GetIPv6().s6_addr, pktinfo->ipi6_addr.s6_addr, + sizeof(pktinfo->ipi6_addr.s6_addr))); + EXPECT_EQ(1, message->msg_iovlen); + iovec iov = message->msg_iov[0]; + EXPECT_EQ(content, std::string(reinterpret_cast(iov.iov_base), iov.iov_len)); } protected: - testing::NiceMock udp_listener_; + testing::NiceMock os_sys_calls_; + TestThreadsafeSingletonInjector os_calls_{&os_sys_calls_}; + testing::NiceMock socket_; quic::QuicIpAddress self_address_; quic::QuicSocketAddress peer_address_; EnvoyQuicPacketWriter envoy_quic_writer_; @@ -47,16 +59,16 @@ TEST_F(EnvoyQuicWriterTest, AssertOnNonNullPacketOption) { EXPECT_DEBUG_DEATH(envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, peer_address_, reinterpret_cast(0x1)), - "Per packet option is not supported yet."); + ""); } TEST_F(EnvoyQuicWriterTest, SendSuccessfully) { std::string str("Hello World!"); - EXPECT_CALL(udp_listener_, send(_)) - .WillOnce(testing::Invoke([this, str](const Network::UdpSendData& send_data) { - verifySendData(str, send_data); - return Api::IoCallUint64Result( - str.length(), Api::IoErrorPtr(nullptr, Network::IoSocketError::deleteIoError)); + + EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) + .WillOnce(testing::Invoke([this, str](int, const msghdr* message, int) { + verifySendData(str, message); + return Api::SysCallSizeResult{static_cast(str.length()), 0}; })); quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, peer_address_, nullptr); @@ -67,12 +79,10 @@ TEST_F(EnvoyQuicWriterTest, SendSuccessfully) { TEST_F(EnvoyQuicWriterTest, SendBlocked) { std::string str("Hello World!"); - EXPECT_CALL(udp_listener_, send(_)) - .WillOnce(testing::Invoke([this, str](const Network::UdpSendData& send_data) { - verifySendData(str, send_data); - return Api::IoCallUint64Result( - 0u, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)); + EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) + .WillOnce(testing::Invoke([this, str](int, const msghdr* message, int) { + verifySendData(str, message); + return Api::SysCallSizeResult{-1, EAGAIN}; })); quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, peer_address_, nullptr); @@ -81,31 +91,26 @@ TEST_F(EnvoyQuicWriterTest, SendBlocked) { EXPECT_TRUE(envoy_quic_writer_.IsWriteBlocked()); // Writing while blocked is not allowed. #ifdef NDEBUG - EXPECT_CALL(udp_listener_, send(_)) - .WillOnce(testing::Invoke([this, str](const Network::UdpSendData& send_data) { - verifySendData(str, send_data); - return Api::IoCallUint64Result( - 0u, Api::IoErrorPtr(Network::IoSocketError::getIoSocketEagainInstance(), - Network::IoSocketError::deleteIoError)); + EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) + .WillOnce(testing::Invoke([this, str](int, const msghdr* message, int) { + verifySendData(str, message); + return Api::SysCallSizeResult{-1, EAGAIN}; })); #endif EXPECT_DEBUG_DEATH(envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, peer_address_, nullptr), - "Cannot write while IO handle is blocked."); + ""); envoy_quic_writer_.SetWritable(); EXPECT_FALSE(envoy_quic_writer_.IsWriteBlocked()); } TEST_F(EnvoyQuicWriterTest, SendFailure) { std::string str("Hello World!"); - EXPECT_CALL(udp_listener_, send(_)) - .WillOnce(testing::Invoke( - [this, str](const Network::UdpSendData& send_data) -> Api::IoCallUint64Result { - verifySendData(str, send_data); - return Api::IoCallUint64Result(0u, - Api::IoErrorPtr(new Network::IoSocketError(ENOTSUP), - Network::IoSocketError::deleteIoError)); - })); + EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) + .WillOnce(testing::Invoke([this, str](int, const msghdr* message, int) { + verifySendData(str, message); + return Api::SysCallSizeResult{-1, ENOTSUP}; + })); quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, peer_address_, nullptr); EXPECT_EQ(quic::WRITE_STATUS_ERROR, result.status); @@ -115,11 +120,10 @@ TEST_F(EnvoyQuicWriterTest, SendFailure) { TEST_F(EnvoyQuicWriterTest, SendFailureMessageTooBig) { std::string str("Hello World!"); - EXPECT_CALL(udp_listener_, send(_)) - .WillOnce(testing::Invoke([this, str](const Network::UdpSendData& send_data) { - verifySendData(str, send_data); - return Api::IoCallUint64Result(0u, Api::IoErrorPtr(new Network::IoSocketError(EMSGSIZE), - Network::IoSocketError::deleteIoError)); + EXPECT_CALL(os_sys_calls_, sendmsg(_, _, _)) + .WillOnce(testing::Invoke([this, str](int, const msghdr* message, int) { + verifySendData(str, message); + return Api::SysCallSizeResult{-1, EMSGSIZE}; })); quic::WriteResult result = envoy_quic_writer_.WritePacket(str.data(), str.length(), self_address_, peer_address_, nullptr); From bb6f945b07eaf04311a04637d201045b1784f909 Mon Sep 17 00:00:00 2001 From: Scott LaVigne <1406859+lavignes@users.noreply.github.com> Date: Tue, 1 Oct 2019 21:23:18 -0700 Subject: [PATCH 31/32] Handle redirects when fetching AWS IAM metadata (#8452) Fixes: #8325 Signed-off-by: Scott LaVigne --- .../filters/http/common/aws/utility.cc | 1 + .../aws_metadata_fetcher_integration_test.cc | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/source/extensions/filters/http/common/aws/utility.cc b/source/extensions/filters/http/common/aws/utility.cc index 0410ef55df92..e8d4df8ebcd4 100644 --- a/source/extensions/filters/http/common/aws/utility.cc +++ b/source/extensions/filters/http/common/aws/utility.cc @@ -110,6 +110,7 @@ absl::optional Utility::metadataFetcher(const std::string& host, curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_TIMEOUT, TIMEOUT.count()); curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); std::string buffer; curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); diff --git a/test/integration/aws_metadata_fetcher_integration_test.cc b/test/integration/aws_metadata_fetcher_integration_test.cc index eb08642bc51a..e542fce7f4e1 100644 --- a/test/integration/aws_metadata_fetcher_integration_test.cc +++ b/test/integration/aws_metadata_fetcher_integration_test.cc @@ -39,6 +39,11 @@ class AwsMetadataIntegrationTestBase : public ::testing::Test, public BaseIntegr virtual_hosts: name: metadata_endpoint routes: + - name: redirect_route + redirect: + prefix_rewrite: "/" + match: + prefix: "/redirect" - name: auth_route direct_response: status: {} @@ -99,6 +104,22 @@ TEST_F(AwsMetadataIntegrationTestSuccess, AuthToken) { EXPECT_EQ(1, test_server_->counter("http.metadata_test.downstream_rq_completed")->value()); } +TEST_F(AwsMetadataIntegrationTestSuccess, Redirect) { + const auto endpoint = fmt::format("{}:{}", Network::Test::getLoopbackAddressUrlString(version_), + lookupPort("listener_0")); + const auto response = Utility::metadataFetcher(endpoint, "redirect", "AUTH_TOKEN"); + + ASSERT_TRUE(response.has_value()); + EXPECT_EQ("METADATA_VALUE_WITH_AUTH", *response); + + // We should make 2 requests, 1 that results in a redirect, and a final successful one + ASSERT_NE(nullptr, test_server_->counter("http.metadata_test.downstream_rq_completed")); + EXPECT_EQ(2, test_server_->counter("http.metadata_test.downstream_rq_completed")->value()); + + ASSERT_NE(nullptr, test_server_->counter("http.metadata_test.downstream_rq_3xx")); + EXPECT_EQ(1, test_server_->counter("http.metadata_test.downstream_rq_3xx")->value()); +} + class AwsMetadataIntegrationTestFailure : public AwsMetadataIntegrationTestBase { public: AwsMetadataIntegrationTestFailure() : AwsMetadataIntegrationTestBase(503, 0) {} From c776fc3bd710d1c4a5e576144986cbff8f013bf4 Mon Sep 17 00:00:00 2001 From: easy Date: Thu, 3 Oct 2019 00:25:50 +1000 Subject: [PATCH 32/32] build: upgrade io_opencensus_cpp. (#8340) Fixes ocagent exporter bug where trace and span IDs were encoded incorrectly. Adds support for service_name. Signed-off-by: Emil Mikulic --- bazel/repository_locations.bzl | 8 ++++---- .../tracers/opencensus/opencensus_tracer_impl.cc | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 62c5a95d3ec0..e950b51ef474 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -253,10 +253,10 @@ REPOSITORY_LOCATIONS = dict( urls = ["https://files.pythonhosted.org/packages/b3/b2/238e2590826bfdd113244a40d9d3eb26918bd798fc187e2360a8367068db/six-1.10.0.tar.gz"], ), io_opencensus_cpp = dict( - sha256 = "29b2be0d92523a20daef7045e547c517ca0f6e6a0ddd7e7fcd15d162c13b6d9a", - strip_prefix = "opencensus-cpp-1bbde06b72e0516b74b6992d204f254073047b10", - # 2019-09-17 - urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/1bbde06b72e0516b74b6992d204f254073047b10.tar.gz"], + sha256 = "c95ab57835182b8b4b17cf5bbfc2406805bc78c5022c17399f3e5c643f22826a", + strip_prefix = "opencensus-cpp-98970f78091ae65b4a029bcf512696ba6d665cf4", + # 2019-09-24 + urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/98970f78091ae65b4a029bcf512696ba6d665cf4.tar.gz"], ), com_github_curl = dict( sha256 = "4376ac72b95572fb6c4fbffefb97c7ea0dd083e1974c0e44cd7e49396f454839", diff --git a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc index a5562d20c81f..ffe49be82159 100644 --- a/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc +++ b/source/extensions/tracers/opencensus/opencensus_tracer_impl.cc @@ -261,6 +261,7 @@ Driver::Driver(const envoy::config::trace::v2::OpenCensusConfig& oc_config, if (oc_config.ocagent_exporter_enabled()) { ::opencensus::exporters::trace::OcAgentOptions opts; opts.address = oc_config.ocagent_address(); + opts.service_name = local_info_.clusterName(); ::opencensus::exporters::trace::OcAgentExporter::Register(std::move(opts)); } }