diff --git a/api/docs/BUILD b/api/docs/BUILD index 0deafc4e0491..df855569bb3b 100644 --- a/api/docs/BUILD +++ b/api/docs/BUILD @@ -18,6 +18,7 @@ proto_library( "//envoy/admin/v2alpha:memory", "//envoy/admin/v2alpha:mutex_stats", "//envoy/admin/v2alpha:server_info", + "//envoy/admin/v2alpha:tap", "//envoy/api/v2:cds", "//envoy/api/v2:discovery", "//envoy/api/v2:eds", @@ -46,6 +47,7 @@ proto_library( "//envoy/config/filter/http/rbac/v2:rbac", "//envoy/config/filter/http/router/v2:router", "//envoy/config/filter/http/squash/v2:squash", + "//envoy/config/filter/http/tap/v2alpha:tap", "//envoy/config/filter/http/transcoder/v2:transcoder", "//envoy/config/filter/listener/original_src/v2alpha1:original_src", "//envoy/config/filter/network/client_ssl_auth/v2:client_ssl_auth", @@ -72,6 +74,8 @@ proto_library( "//envoy/data/accesslog/v2:accesslog", "//envoy/data/core/v2alpha:health_check_event", "//envoy/data/tap/v2alpha:capture", + "//envoy/data/tap/v2alpha:http", + "//envoy/data/tap/v2alpha:wrapper", "//envoy/service/accesslog/v2:als", "//envoy/service/auth/v2alpha:attribute_context", "//envoy/service/auth/v2alpha:external_auth", @@ -79,6 +83,7 @@ proto_library( "//envoy/service/load_stats/v2:lrs", "//envoy/service/metrics/v2:metrics_service", "//envoy/service/ratelimit/v2:rls", + "//envoy/service/tap/v2alpha:common", "//envoy/type:percent", "//envoy/type:range", "//envoy/type/matcher:metadata", diff --git a/api/envoy/admin/v2alpha/BUILD b/api/envoy/admin/v2alpha/BUILD index 332c07058bb6..a6b403fdd23e 100644 --- a/api/envoy/admin/v2alpha/BUILD +++ b/api/envoy/admin/v2alpha/BUILD @@ -55,3 +55,11 @@ api_proto_library_internal( srcs = ["server_info.proto"], visibility = ["//visibility:public"], ) + +api_proto_library_internal( + name = "tap", + srcs = ["tap.proto"], + deps = [ + "//envoy/service/tap/v2alpha:common", + ], +) diff --git a/api/envoy/admin/v2alpha/tap.proto b/api/envoy/admin/v2alpha/tap.proto new file mode 100644 index 000000000000..e35ef3cc5aec --- /dev/null +++ b/api/envoy/admin/v2alpha/tap.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +import "envoy/service/tap/v2alpha/common.proto"; +import "validate/validate.proto"; + +package envoy.admin.v2alpha; +option java_package = "io.envoyproxy.envoy.admin.v2alpha"; + +// The /tap admin request body that is used to configure an active tap session. +message TapRequest { + // The opaque configuration ID used to match the configuration to a loaded extension. + // A tap extension configures a similar opaque ID that is used to match. + string config_id = 1 [(validate.rules).string.min_bytes = 1]; + + // The tap configuration to load. + service.tap.v2alpha.TapConfig tap_config = 2 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/filter/http/tap/v2alpha/BUILD b/api/envoy/config/filter/http/tap/v2alpha/BUILD new file mode 100644 index 000000000000..acec4094f172 --- /dev/null +++ b/api/envoy/config/filter/http/tap/v2alpha/BUILD @@ -0,0 +1,11 @@ +load("//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "tap", + srcs = ["tap.proto"], + deps = [ + "//envoy/service/tap/v2alpha:common", + ], +) diff --git a/api/envoy/config/filter/http/tap/v2alpha/tap.proto b/api/envoy/config/filter/http/tap/v2alpha/tap.proto new file mode 100644 index 000000000000..8018d5f07f92 --- /dev/null +++ b/api/envoy/config/filter/http/tap/v2alpha/tap.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +import "envoy/service/tap/v2alpha/common.proto"; + +import "validate/validate.proto"; + +package envoy.config.filter.http.tap.v2alpha; +option java_package = "io.envoyproxy.envoy.config.filter.http.tap.v2alpha"; + +// [#protodoc-title: Tap] +// Tap :ref:`configuration overview `. + +// Top level configuration for the tap filter. +message Tap { + oneof config_type { + option (validate.required) = true; + + // If specified, the tap filter will be configured via an admin handler. + AdminConfig admin_config = 1; + } +} + +// Configuration for the admin handler. See :ref:`here ` for +// more information. +message AdminConfig { + // Opaque configuration ID. When requests are made to the admin handler, the passed opaque ID is + // matched to the configured filter opaque ID to determine which filter to configure. + string config_id = 1 [(validate.rules).string.min_bytes = 1]; +} diff --git a/api/envoy/data/tap/v2alpha/BUILD b/api/envoy/data/tap/v2alpha/BUILD index 46de68e3a825..97c4a0b82827 100644 --- a/api/envoy/data/tap/v2alpha/BUILD +++ b/api/envoy/data/tap/v2alpha/BUILD @@ -7,3 +7,15 @@ api_proto_library_internal( srcs = ["capture.proto"], deps = ["//envoy/api/v2/core:address"], ) + +api_proto_library_internal( + name = "http", + srcs = ["http.proto"], + deps = ["//envoy/api/v2/core:base"], +) + +api_proto_library_internal( + name = "wrapper", + srcs = ["wrapper.proto"], + deps = [":http"], +) diff --git a/api/envoy/data/tap/v2alpha/capture.proto b/api/envoy/data/tap/v2alpha/capture.proto index 36f6e9b0b8b4..aef84ce14b85 100644 --- a/api/envoy/data/tap/v2alpha/capture.proto +++ b/api/envoy/data/tap/v2alpha/capture.proto @@ -1,6 +1,6 @@ syntax = "proto3"; -// [#protodoc-title: Common TAP] +// [#protodoc-title: Common tap] // Trace capture format for the capture transport socket extension. This dumps plain text read/write // sequences on a socket. diff --git a/api/envoy/data/tap/v2alpha/http.proto b/api/envoy/data/tap/v2alpha/http.proto new file mode 100644 index 000000000000..f9c4bb483ad7 --- /dev/null +++ b/api/envoy/data/tap/v2alpha/http.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package envoy.data.tap.v2alpha; +option java_package = "io.envoyproxy.envoy.data.tap.v2alpha"; + +import "envoy/api/v2/core/base.proto"; + +// [#protodoc-title: HTTP tap data] + +// A fully buffered HTTP trace message. +message HttpBufferedTrace { + // Request headers. + repeated api.v2.core.HeaderValue request_headers = 2; + + // Response headers. + repeated api.v2.core.HeaderValue response_headers = 3; +} diff --git a/api/envoy/data/tap/v2alpha/wrapper.proto b/api/envoy/data/tap/v2alpha/wrapper.proto new file mode 100644 index 000000000000..12a527e866f8 --- /dev/null +++ b/api/envoy/data/tap/v2alpha/wrapper.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +import "envoy/data/tap/v2alpha/http.proto"; + +import "validate/validate.proto"; + +package envoy.data.tap.v2alpha; +option java_package = "io.envoyproxy.envoy.data.tap.v2alpha"; + +// [#protodoc-title: Tap data wrappers] + +// Wrapper for all fully buffered tap traces that Envoy emits. This is required for sending traces +// over gRPC APIs or more easily persisting binary messages to files. +message BufferedTraceWrapper { + oneof trace { + option (validate.required) = true; + + // An HTTP buffered tap trace. + HttpBufferedTrace http_buffered_trace = 1; + } +} diff --git a/api/envoy/service/tap/v2alpha/BUILD b/api/envoy/service/tap/v2alpha/BUILD new file mode 100644 index 000000000000..0d46bdac25a8 --- /dev/null +++ b/api/envoy/service/tap/v2alpha/BUILD @@ -0,0 +1,12 @@ +load("//bazel:api_build_system.bzl", "api_proto_library_internal") + +licenses(["notice"]) # Apache 2 + +api_proto_library_internal( + name = "common", + srcs = ["common.proto"], + visibility = ["//visibility:public"], + deps = [ + "//envoy/api/v2/route", + ], +) diff --git a/api/envoy/service/tap/v2alpha/common.proto b/api/envoy/service/tap/v2alpha/common.proto new file mode 100644 index 000000000000..a70bacf50d78 --- /dev/null +++ b/api/envoy/service/tap/v2alpha/common.proto @@ -0,0 +1,99 @@ +syntax = "proto3"; + +import "envoy/api/v2/route/route.proto"; + +import "validate/validate.proto"; + +package envoy.service.tap.v2alpha; +option java_package = "io.envoyproxy.envoy.service.tap.v2alpha"; + +// [#protodoc-title: Common tap configuration] + +// Tap configuration. +message TapConfig { + // The match configuration. If the configuration matches the data source being tapped, a tap will + // occur, with the result written to the configured output. + MatchPredicate match_config = 1 [(validate.rules).message.required = true]; + + // The tap output configuration. If a match configuration matches a data source being tapped, + // a tap will occur and the data will be written to the configured output. + OutputConfig output_config = 2 [(validate.rules).message.required = true]; + + // [#comment:TODO(mattklein123): Rate limiting] +} + +// Tap match configuration. This is a recursive structure which allows complex nested match +// configurations to be built using various logical operators. +message MatchPredicate { + // A set of match configurations used for logical operations. + message MatchSet { + // The list of rules that make up the set. + repeated MatchPredicate rules = 1 [(validate.rules).repeated .min_items = 2]; + } + + oneof rule { + option (validate.required) = true; + + // A set that describes a logical OR. If any member of the set matches, the match configuration + // matches. + MatchSet or_match = 1; + + // A set that describes a logical AND. If all members of the set match, the match configuration + // matches. + MatchSet and_match = 2; + + // A negation match. The match configuration will match if the negated match condition matches. + MatchPredicate not_match = 3; + + // The match configuration will always match. + bool any_match = 4 [(validate.rules).bool.const = true]; + + // HTTP request match configuration. + HttpRequestMatch http_request_match = 5; + + // HTTP response match configuration. + HttpResponseMatch http_response_match = 6; + } +} + +// HTTP request match configuration. +message HttpRequestMatch { + // HTTP request headers to match. + repeated api.v2.route.HeaderMatcher headers = 1; +} + +// HTTP response match configuration. +message HttpResponseMatch { + // HTTP response headers to match. + repeated api.v2.route.HeaderMatcher headers = 1; +} + +// Tap output configuration. +message OutputConfig { + // Output sinks for tap data. Currently a single sink is allowed in the list. Once multiple + // sink types are supported this constraint will be relaxed. + repeated OutputSink sinks = 1 [(validate.rules).repeated = {min_items: 1, max_items: 1}]; + + // [#comment:TODO(mattklein123): Output filtering. E.g., certain headers, truncated body, etc.] +} + +// Tap output sink configuration. +message OutputSink { + oneof output_sink_type { + option (validate.required) = true; + + // Tap output will be streamed out the :http:post:`/tap` admin endpoint. + // + // .. attention:: + // + // It is only allowed to specify the streaming admin output sink if the tap is being + // configured from the :http:post:`/tap` admin endpoint. Thus, if an extension has + // been configured to receive tap configuration from some other source (e.g., static + // file, XDS, etc.) configuring the streaming admin output type will fail. + StreamingAdminSink streaming_admin = 1; + } +} + +// Streaming admin sink configuration. +message StreamingAdminSink { +} diff --git a/docs/build.sh b/docs/build.sh index e121fb52eacc..bf5fafcf31d4 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -54,6 +54,7 @@ PROTO_RST=" /envoy/admin/v2alpha/clusters/envoy/admin/v2alpha/metrics.proto.rst /envoy/admin/v2alpha/mutex_stats/envoy/admin/v2alpha/mutex_stats.proto.rst /envoy/admin/v2alpha/server_info/envoy/admin/v2alpha/server_info.proto.rst + /envoy/admin/v2alpha/tap/envoy/admin/v2alpha/tap.proto.rst /envoy/api/v2/core/address/envoy/api/v2/core/address.proto.rst /envoy/api/v2/core/base/envoy/api/v2/core/base.proto.rst /envoy/api/v2/core/http_uri/envoy/api/v2/core/http_uri.proto.rst @@ -95,6 +96,7 @@ PROTO_RST=" /envoy/config/filter/http/rbac/v2/rbac/envoy/config/filter/http/rbac/v2/rbac.proto.rst /envoy/config/filter/http/router/v2/router/envoy/config/filter/http/router/v2/router.proto.rst /envoy/config/filter/http/squash/v2/squash/envoy/config/filter/http/squash/v2/squash.proto.rst + /envoy/config/filter/http/tap/v2alpha/tap/envoy/config/filter/http/tap/v2alpha/tap.proto.rst /envoy/config/filter/http/transcoder/v2/transcoder/envoy/config/filter/http/transcoder/v2/transcoder.proto.rst /envoy/config/filter/listener/original_src/v2alpha1/original_src/envoy/config/filter/listener/original_src/v2alpha1/original_src.proto.rst /envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth/envoy/config/filter/network/client_ssl_auth/v2/client_ssl_auth.proto.rst @@ -118,10 +120,13 @@ PROTO_RST=" /envoy/data/accesslog/v2/accesslog/envoy/data/accesslog/v2/accesslog.proto.rst /envoy/data/core/v2alpha/health_check_event/envoy/data/core/v2alpha/health_check_event.proto.rst /envoy/data/tap/v2alpha/capture/envoy/data/tap/v2alpha/capture.proto.rst + /envoy/data/tap/v2alpha/http/envoy/data/tap/v2alpha/http.proto.rst + /envoy/data/tap/v2alpha/wrapper/envoy/data/tap/v2alpha/wrapper.proto.rst /envoy/service/accesslog/v2/als/envoy/service/accesslog/v2/als.proto.rst /envoy/service/auth/v2alpha/external_auth/envoy/service/auth/v2alpha/attribute_context.proto.rst /envoy/service/auth/v2alpha/external_auth/envoy/service/auth/v2alpha/external_auth.proto.rst /envoy/service/ratelimit/v2/rls/envoy/service/ratelimit/v2/rls.proto.rst + /envoy/service/tap/v2alpha/common/envoy/service/tap/v2alpha/common.proto.rst /envoy/type/http_status/envoy/type/http_status.proto.rst /envoy/type/percent/envoy/type/percent.proto.rst /envoy/type/range/envoy/type/range.proto.rst diff --git a/docs/root/api-v2/admin/admin.rst b/docs/root/api-v2/admin/admin.rst index 4837d9d10a7a..0f5579251d93 100644 --- a/docs/root/api-v2/admin/admin.rst +++ b/docs/root/api-v2/admin/admin.rst @@ -5,10 +5,4 @@ Admin :glob: :maxdepth: 2 - ../admin/v2alpha/certs.proto - ../admin/v2alpha/config_dump.proto - ../admin/v2alpha/clusters.proto - ../admin/v2alpha/memory.proto - ../admin/v2alpha/metrics.proto - ../admin/v2alpha/mutex_stats.proto - ../admin/v2alpha/server_info.proto + ../admin/v2alpha/* diff --git a/docs/root/api-v2/data/tap/tap.rst b/docs/root/api-v2/data/tap/tap.rst index b6514156bcab..b1e7b3dd5017 100644 --- a/docs/root/api-v2/data/tap/tap.rst +++ b/docs/root/api-v2/data/tap/tap.rst @@ -5,4 +5,4 @@ Tap :glob: :maxdepth: 2 - v2alpha/capture.proto + v2alpha/* diff --git a/docs/root/api-v2/service/service.rst b/docs/root/api-v2/service/service.rst index 00d2698ca161..4cf1c5e9f284 100644 --- a/docs/root/api-v2/service/service.rst +++ b/docs/root/api-v2/service/service.rst @@ -7,3 +7,4 @@ Services accesslog/v2/* ratelimit/v2/* + tap/v2alpha/* diff --git a/docs/root/configuration/http_filters/http_filters.rst b/docs/root/configuration/http_filters/http_filters.rst index 75be91b9f844..6c2d38b8c81e 100644 --- a/docs/root/configuration/http_filters/http_filters.rst +++ b/docs/root/configuration/http_filters/http_filters.rst @@ -25,3 +25,4 @@ HTTP filters rbac_filter router_filter squash_filter + tap_filter diff --git a/docs/root/configuration/http_filters/tap_filter.rst b/docs/root/configuration/http_filters/tap_filter.rst new file mode 100644 index 000000000000..f2fe8bd83ff0 --- /dev/null +++ b/docs/root/configuration/http_filters/tap_filter.rst @@ -0,0 +1,148 @@ +.. _config_http_filters_tap: + +Tap +=== + +* :ref:`v2 API reference ` +* This filter should be configured with the name *envoy.filters.http.tap*. + +.. attention:: + + The tap filter is experimental and is currently under active development. There is currently a + very limited set of match conditions, output configuration, output sinks, etc. Capabilities will + be expanded over time and the configuration structures are likely to change. + +The HTTP tap filter is used to interpose on and record HTTP traffic. At a high level, the +configuration is composed of two pieces: + +1. :ref:`Match configuration `: a list of + conditions under which the filter will match an HTTP request and begin a tap session. +2. :ref:`Output configuration `: a list of output + sinks that the filter will write the matched and tapped data to. + +Each of these concepts will be covered incrementally over the course of several example +configurations in the following section. + +Example configuration +--------------------- + +Example filter configuration: + +.. code-block:: yaml + + name: envoy.filters.http.tap + config: + admin_config: + config_id: test_config_id + +The previous snippet configures the filter for control via the :http:post:`/tap` admin handler. +See the following section for more details. + +.. _config_http_filters_tap_admin_handler: + +Admin handler +------------- + +When the HTTP filter specifies an :ref:`admin_config +`, it is configured for admin control and +the :http:post:`/tap` admin handler will be installed. The admin handler can be used for live +tapping and debugging of HTTP traffic. It works as follows: + +1. A POST request is used to provide a valid tap configuration. The POST request body can be either + the JSON or YAML representation of the :ref:`TapConfig + ` message. +2. If the POST request is accepted, Envoy will stream :ref:`HttpBufferedTrace + ` messages (serialized to JSON) until the admin + request is terminated. + +An example POST body: + +.. code-block:: yaml + + config_id: test_config_id + tap_config: + match_config: + and_match: + rules: + - http_request_match: + headers: + - name: foo + exact_match: bar + - http_response_match: + headers: + - name: bar + exact_match: baz + output_config: + sinks: + - streaming_admin: {} + +The preceding configuration instructs the tap filter to match any HTTP requests in which a request +header ``foo: bar`` is present AND a response header ``bar: baz`` is present. If both of these +conditions are met, the request will be tapped and streamed out the admin endpoint. + +Another example POST body: + +.. code-block:: yaml + + config_id: test_config_id + tap_config: + match_config: + or_match: + rules: + - http_request_match: + headers: + - name: foo + exact_match: bar + - http_response_match: + headers: + - name: bar + exact_match: baz + output_config: + sinks: + - streaming_admin: {} + +The preceding configuration instructs the tap filter to match any HTTP requests in which a request +header ``foo: bar`` is present OR a response header ``bar: baz`` is present. If either of these +conditions are met, the request will be tapped and streamed out the admin endpoint. + +Another example POST body: + +.. code-block:: yaml + + config_id: test_config_id + tap_config: + match_config: + any_match: true + output_config: + sinks: + - streaming_admin: {} + +The preceding configuration instructs the tap filter to match any HTTP requests. All requests will +be tapped and streamed out the admin endpoint. + +Streaming matching +------------------ + +The tap filter supports "streaming matching." This means that instead of waiting until the end of +the request/response sequence, the filter will match incrementally as the request proceeds. I.e., +first the request headers will be matched, then the request body if present, then the request +trailers if present, then the response headers if present, etc. + +In the future, the filter will support streaming output. Currently only :ref:`fully buffered output +` is implemented. However, even in the current +implementation, if a tap is configured to match request headers and the request headers match, +even if there is no response (upstream failure, etc.) the request will still be tapped and sent +to the configured output. + +Statistics +---------- + +The tap filter outputs statistics in the *http..tap.* namespace. The :ref:`stat prefix +` +comes from the owning HTTP connection manager. + +.. csv-table:: + :header: Name, Type, Description + :widths: 1, 1, 2 + + rq_tapped, Counter, Total requests that matched and were tapped diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 62063ff13600..d17d96fa9b74 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -3,11 +3,11 @@ Version history 1.10.0 (pending) ================ -* config: added support of using google.protobuf.Any in opaque configs for extensions. -* config: removed deprecated --v2-config-only from command line config. * access log: added a new flag for upstream retry count exceeded. * admin: the admin server can now be accessed via HTTP/2 (prior knowledge). -* buffer: fix vulnerabilities when allocation fails +* buffer: fix vulnerabilities when allocation fails. +* config: added support of using google.protobuf.Any in opaque configs for extensions. +* config: removed deprecated --v2-config-only from command line config. * config: removed deprecated_v1 sds_config from :ref:`Bootstrap config `. * config: removed REST_LEGACY as a valid :ref:`ApiType `. * cors: added :ref:`filter_enabled & shadow_enabled RuntimeFractionalPercent flags ` to filter. @@ -22,6 +22,7 @@ Version history * redis: added :ref:`success and error stats ` for commands. * router: added ability to configure a :ref:`retry policy ` at the virtual host level. +* tap: added new alpha :ref:`HTTP tap filter `. * tls: enabled TLS 1.3 on the server-side (non-FIPS builds). * router: added per-route configuration of :ref:`internal redirects `. * upstream: add hash_function to specify the hash function for :ref:`ring hash` as either xxHash or `murmurHash2 `_. MurmurHash2 is compatible with std::hash in GNU libstdc++ 3.4.20 or above. This is typically the case when compiled on Linux and not macOS. diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index a8016befefe7..032ee4a3090d 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -399,3 +399,11 @@ explanation of the output. set to '0'. * Latency information represents data since last flush. Mean latency is currently not available. + +.. http:post:: /tap + + This endpoint is used for configuring an active tap session. It is only + available if a valid tap extension has been configured, and that extension has + been configured to accept admin configuration. See: + + * :ref:`HTTP tap filter configuration ` diff --git a/source/common/common/logger.h b/source/common/common/logger.h index 68587928f0c2..0b645cb34d7d 100644 --- a/source/common/common/logger.h +++ b/source/common/common/logger.h @@ -47,6 +47,7 @@ namespace Logger { FUNCTION(runtime) \ FUNCTION(stats) \ FUNCTION(secret) \ + FUNCTION(tap) \ FUNCTION(testing) \ FUNCTION(thrift) \ FUNCTION(tracing) \ diff --git a/source/extensions/common/tap/BUILD b/source/extensions/common/tap/BUILD new file mode 100644 index 000000000000..5362c729b470 --- /dev/null +++ b/source/extensions/common/tap/BUILD @@ -0,0 +1,52 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "tap_interface", + hdrs = ["tap.h"], + deps = [ + "//include/envoy/http:header_map_interface", + "@envoy_api//envoy/data/tap/v2alpha:wrapper_cc", + "@envoy_api//envoy/service/tap/v2alpha:common_cc", + ], +) + +envoy_cc_library( + name = "tap_config_base", + srcs = ["tap_config_base.cc"], + hdrs = ["tap_config_base.h"], + deps = [ + ":tap_interface", + ":tap_matcher", + "//source/common/common:assert_lib", + ], +) + +envoy_cc_library( + name = "tap_matcher", + srcs = ["tap_matcher.cc"], + hdrs = ["tap_matcher.h"], + deps = [ + "//source/common/http:header_utility_lib", + "@envoy_api//envoy/service/tap/v2alpha:common_cc", + ], +) + +envoy_cc_library( + name = "admin", + srcs = ["admin.cc"], + hdrs = ["admin.h"], + deps = [ + ":tap_interface", + "//include/envoy/server:admin_interface", + "//include/envoy/singleton:manager_interface", + "@envoy_api//envoy/admin/v2alpha:tap_cc", + ], +) diff --git a/source/extensions/common/tap/admin.cc b/source/extensions/common/tap/admin.cc new file mode 100644 index 000000000000..d2a0369c21bb --- /dev/null +++ b/source/extensions/common/tap/admin.cc @@ -0,0 +1,116 @@ +#include "extensions/common/tap/admin.h" + +#include "envoy/admin/v2alpha/tap.pb.h" +#include "envoy/admin/v2alpha/tap.pb.validate.h" + +#include "common/buffer/buffer_impl.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Tap { + +// Singleton registration via macro defined in envoy/singleton/manager.h +SINGLETON_MANAGER_REGISTRATION(tap_admin_handler); + +AdminHandlerSharedPtr AdminHandler::getSingleton(Server::Admin& admin, + Singleton::Manager& singleton_manager, + Event::Dispatcher& main_thread_dispatcher) { + return singleton_manager.getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(tap_admin_handler), [&admin, &main_thread_dispatcher] { + return std::make_shared(admin, main_thread_dispatcher); + }); +} + +AdminHandler::AdminHandler(Server::Admin& admin, Event::Dispatcher& main_thread_dispatcher) + : admin_(admin), main_thread_dispatcher_(main_thread_dispatcher) { + const bool rc = + admin_.addHandler("/tap", "tap filter control", MAKE_ADMIN_HANDLER(handler), true, true); + RELEASE_ASSERT(rc, "/tap admin endpoint is taken"); +} + +AdminHandler::~AdminHandler() { + const bool rc = admin_.removeHandler("/tap"); + ASSERT(rc); +} + +Http::Code AdminHandler::handler(absl::string_view, Http::HeaderMap&, Buffer::Instance& response, + Server::AdminStream& admin_stream) { + if (attached_request_.has_value()) { + // TODO(mattlklein123): Consider supporting concurrent admin /tap streams. Right now we support + // a single stream as a simplification. + return badRequest(response, "An attached /tap admin stream already exists. Detach it."); + } + + if (admin_stream.getRequestBody() == nullptr) { + return badRequest(response, "/tap requires a JSON/YAML body"); + } + + envoy::admin::v2alpha::TapRequest tap_request; + try { + MessageUtil::loadFromYamlAndValidate(admin_stream.getRequestBody()->toString(), tap_request); + } catch (EnvoyException& e) { + return badRequest(response, e.what()); + } + + ENVOY_LOG(debug, "tap admin request for config_id={}", tap_request.config_id()); + if (config_id_map_.count(tap_request.config_id()) == 0) { + return badRequest( + response, fmt::format("Unknown config id '{}'. No extension has registered with this id.", + tap_request.config_id())); + } + for (auto config : config_id_map_[tap_request.config_id()]) { + config->newTapConfig(std::move(*tap_request.mutable_tap_config()), this); + } + + admin_stream.setEndStreamOnComplete(false); + admin_stream.addOnDestroyCallback([this] { + for (auto config : config_id_map_[attached_request_.value().config_id_]) { + ENVOY_LOG(debug, "detach tap admin request for config_id={}", + attached_request_.value().config_id_); + config->clearTapConfig(); + attached_request_ = absl::nullopt; + } + }); + attached_request_.emplace(tap_request.config_id(), &admin_stream); + return Http::Code::OK; +} + +Http::Code AdminHandler::badRequest(Buffer::Instance& response, absl::string_view error) { + ENVOY_LOG(debug, "handler bad request: {}", error); + response.add(error); + return Http::Code::BadRequest; +} + +void AdminHandler::registerConfig(ExtensionConfig& config, const std::string& config_id) { + ASSERT(!config_id.empty()); + ASSERT(config_id_map_[config_id].count(&config) == 0); + config_id_map_[config_id].insert(&config); +} + +void AdminHandler::unregisterConfig(ExtensionConfig& config) { + ASSERT(!config.adminId().empty()); + ASSERT(config_id_map_[config.adminId()].count(&config) == 1); + config_id_map_[config.adminId()].erase(&config); + if (config_id_map_[config.adminId()].empty()) { + config_id_map_.erase(config.adminId()); + } +} + +void AdminHandler::submitBufferedTrace( + std::shared_ptr trace) { + ENVOY_LOG(debug, "admin submitting buffered trace to main thread"); + main_thread_dispatcher_.post([this, trace]() { + if (attached_request_.has_value()) { + ENVOY_LOG(debug, "admin writing buffered trace to response"); + Buffer::OwnedImpl json_trace{MessageUtil::getJsonStringFromMessage(*trace, true, true)}; + attached_request_.value().admin_stream_->getDecoderFilterCallbacks().encodeData(json_trace, + false); + } + }); +} + +} // namespace Tap +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/tap/admin.h b/source/extensions/common/tap/admin.h new file mode 100644 index 000000000000..b86cc1e232f8 --- /dev/null +++ b/source/extensions/common/tap/admin.h @@ -0,0 +1,78 @@ +#pragma once + +#include "envoy/server/admin.h" +#include "envoy/singleton/manager.h" + +#include "extensions/common/tap/tap.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Tap { + +class AdminHandler; +using AdminHandlerSharedPtr = std::shared_ptr; + +/** + * Singleton /tap admin handler for admin management of tap configurations and output. This + * handler is not installed and active unless the tap configuration specifically configures it. + * TODO(mattklein123): We should allow the admin handler to always be installed in read only mode + * so it's easier to debug the active tap configuration. + */ +class AdminHandler : public Singleton::Instance, + public Extensions::Common::Tap::Sink, + Logger::Loggable { +public: + AdminHandler(Server::Admin& admin, Event::Dispatcher& main_thread_dispatcher); + ~AdminHandler() override; + + /** + * Get the singleton admin handler. The handler will be created if it doesn't already exist, + * otherwise the existing handler will be returned. + */ + static AdminHandlerSharedPtr getSingleton(Server::Admin& admin, + Singleton::Manager& singleton_manager, + Event::Dispatcher& main_thread_dispatcher); + + /** + * Register a new extension config to the handler so that it can be admin managed. + * @param config supplies the config to register. + * @param config_id supplies the ID to use for managing the configuration. Multiple extensions + * can use the same ID so they can be managed in aggregate (e.g., an HTTP filter on + * many listeners). + */ + void registerConfig(ExtensionConfig& config, const std::string& config_id); + + /** + * Unregister an extension config from the handler. + * @param config supplies the previously registered config. + */ + void unregisterConfig(ExtensionConfig& config); + + // Extensions::Common::Tap::Sink + void submitBufferedTrace( + std::shared_ptr trace) override; + +private: + struct AttachedRequest { + AttachedRequest(std::string config_id, Server::AdminStream* admin_stream) + : config_id_(std::move(config_id)), admin_stream_(admin_stream) {} + + const std::string config_id_; + const Server::AdminStream* admin_stream_; + }; + + Http::Code handler(absl::string_view path_and_query, Http::HeaderMap& response_headers, + Buffer::Instance& response, Server::AdminStream& admin_stream); + Http::Code badRequest(Buffer::Instance& response, absl::string_view error); + + Server::Admin& admin_; + Event::Dispatcher& main_thread_dispatcher_; + std::unordered_map> config_id_map_; + absl::optional attached_request_; +}; + +} // namespace Tap +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/tap/tap.h b/source/extensions/common/tap/tap.h new file mode 100644 index 000000000000..4d2d6bea469b --- /dev/null +++ b/source/extensions/common/tap/tap.h @@ -0,0 +1,62 @@ +#pragma once + +#include "envoy/common/pure.h" +#include "envoy/data/tap/v2alpha/wrapper.pb.h" +#include "envoy/http/header_map.h" +#include "envoy/service/tap/v2alpha/common.pb.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Tap { + +/** + * Sink for sending tap messages. + */ +class Sink { +public: + virtual ~Sink() = default; + + /** + * Send a fully buffered trace to the sink. + * @param trace supplies the trace to send. The trace message is a discrete trace message (as + * opposed to a portion of a larger trace that should be aggregated). + */ + virtual void + submitBufferedTrace(std::shared_ptr trace) PURE; +}; + +/** + * Generic configuration for a tap extension (filter, transport socket, etc.). + */ +class ExtensionConfig { +public: + virtual ~ExtensionConfig() = default; + + /** + * @return the ID to use for admin extension configuration tracking (if applicable). + */ + virtual const std::string& adminId() PURE; + + /** + * Clear any active tap configuration. + */ + virtual void clearTapConfig() PURE; + + /** + * Install a new tap configuration. + * @param proto_config supplies the generic tap config to install. Not all configuration fields + * may be applicable to an extension (e.g. HTTP fields). The extension is free to fail + * the configuration load via exception if it wishes. + * @param admin_streamer supplies the singleton admin sink to use for output if the configuration + * specifies that output type. May not be used if the configuration does not specify + * admin output. May be nullptr if admin is not used to supply the config. + */ + virtual void newTapConfig(envoy::service::tap::v2alpha::TapConfig&& proto_config, + Sink* admin_streamer) PURE; +}; + +} // namespace Tap +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/tap/tap_config_base.cc b/source/extensions/common/tap/tap_config_base.cc new file mode 100644 index 000000000000..3bcb95b04ed6 --- /dev/null +++ b/source/extensions/common/tap/tap_config_base.cc @@ -0,0 +1,32 @@ +#include "extensions/common/tap/tap_config_base.h" + +#include "common/common/assert.h" + +#include "extensions/common/tap/tap_matcher.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Tap { + +TapConfigBaseImpl::TapConfigBaseImpl(envoy::service::tap::v2alpha::TapConfig&& proto_config, + Common::Tap::Sink* admin_streamer) + : admin_streamer_(admin_streamer) { + + // TODO(mattklein123): The streaming admin output sink is the only currently supported sink. This + // is validated by schema. + ASSERT(admin_streamer != nullptr); + ASSERT(proto_config.output_config().sinks()[0].has_streaming_admin()); + + buildMatcher(proto_config.match_config(), matchers_); +} + +Matcher& TapConfigBaseImpl::rootMatcher() { + ASSERT(matchers_.size() >= 1); + return *matchers_[0]; +} + +} // namespace Tap +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/tap/tap_config_base.h b/source/extensions/common/tap/tap_config_base.h new file mode 100644 index 000000000000..d273b28a2c94 --- /dev/null +++ b/source/extensions/common/tap/tap_config_base.h @@ -0,0 +1,39 @@ +#pragma once + +#include "envoy/service/tap/v2alpha/common.pb.h" + +#include "extensions/common/tap/tap.h" +#include "extensions/common/tap/tap_matcher.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Tap { + +/** + * Base class for all tap configurations. + * TODO(mattklein123): This class will handle common functionality such as rate limiting, etc. + */ +class TapConfigBaseImpl { +public: + size_t numMatchers() { return matchers_.size(); } + Matcher& rootMatcher(); + Extensions::Common::Tap::Sink& sink() { + // TODO(mattklein123): When we support multiple sinks, select the right one. Right now + // it must be admin. + return *admin_streamer_; + } + +protected: + TapConfigBaseImpl(envoy::service::tap::v2alpha::TapConfig&& proto_config, + Common::Tap::Sink* admin_streamer); + +private: + Sink* admin_streamer_; + std::vector matchers_; +}; + +} // namespace Tap +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/tap/tap_matcher.cc b/source/extensions/common/tap/tap_matcher.cc new file mode 100644 index 000000000000..33d77c927bcc --- /dev/null +++ b/source/extensions/common/tap/tap_matcher.cc @@ -0,0 +1,133 @@ +#include "extensions/common/tap/tap_matcher.h" + +#include "common/common/assert.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Tap { + +void buildMatcher(const envoy::service::tap::v2alpha::MatchPredicate& match_config, + std::vector& matchers) { + // In order to store indexes and build our matcher tree inline, we must reserve a slot where + // the matcher we are about to create will go. This allows us to know its future index and still + // construct more of the tree in each called constructor (e.g., multiple OR/AND conditions). + // Once fully constructed, we move the matcher into its position below. See the tap matcher + // overview in tap.h for more information. + matchers.emplace_back(nullptr); + + MatcherPtr new_matcher; + switch (match_config.rule_case()) { + case envoy::service::tap::v2alpha::MatchPredicate::kOrMatch: + new_matcher = std::make_unique(match_config.or_match(), matchers, + SetLogicMatcher::Type::Or); + break; + case envoy::service::tap::v2alpha::MatchPredicate::kAndMatch: + new_matcher = std::make_unique(match_config.and_match(), matchers, + SetLogicMatcher::Type::And); + break; + case envoy::service::tap::v2alpha::MatchPredicate::kNotMatch: + new_matcher = std::make_unique(match_config.not_match(), matchers); + break; + case envoy::service::tap::v2alpha::MatchPredicate::kAnyMatch: + new_matcher = std::make_unique(matchers); + break; + case envoy::service::tap::v2alpha::MatchPredicate::kHttpRequestMatch: + new_matcher = std::make_unique(match_config.http_request_match(), matchers); + break; + case envoy::service::tap::v2alpha::MatchPredicate::kHttpResponseMatch: + new_matcher = + std::make_unique(match_config.http_response_match(), matchers); + break; + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } + + // Per above, move the matcher into its position. + matchers[new_matcher->index()] = std::move(new_matcher); +} + +SetLogicMatcher::SetLogicMatcher( + const envoy::service::tap::v2alpha::MatchPredicate::MatchSet& configs, + std::vector& matchers, Type type) + : Matcher(matchers), matchers_(matchers), type_(type) { + for (const auto& config : configs.rules()) { + indexes_.push_back(matchers_.size()); + buildMatcher(config, matchers_); + } +} + +bool SetLogicMatcher::updateMatchStatus(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + std::vector& statuses) const { + for (size_t index : indexes_) { + statuses[index] = + matchers_[index]->updateMatchStatus(request_headers, response_headers, statuses); + } + + auto predicate = [&statuses](size_t index) { return statuses[index]; }; + if (type_ == Type::And) { + statuses[my_index_] = std::all_of(indexes_.begin(), indexes_.end(), predicate); + } else { + ASSERT(type_ == Type::Or); + statuses[my_index_] = std::any_of(indexes_.begin(), indexes_.end(), predicate); + } + + return statuses[my_index_]; +} + +NotMatcher::NotMatcher(const envoy::service::tap::v2alpha::MatchPredicate& config, + std::vector& matchers) + : Matcher(matchers), matchers_(matchers), not_index_(matchers.size()) { + buildMatcher(config, matchers); +} + +bool NotMatcher::updateMatchStatus(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + std::vector& statuses) const { + statuses[my_index_] = + !matchers_[not_index_]->updateMatchStatus(request_headers, response_headers, statuses); + return statuses[my_index_]; +} + +HttpRequestMatcher::HttpRequestMatcher(const envoy::service::tap::v2alpha::HttpRequestMatch& config, + const std::vector& matchers) + : Matcher(matchers) { + for (const auto& header_match : config.headers()) { + headers_to_match_.emplace_back(header_match); + } +} + +bool HttpRequestMatcher::updateMatchStatus(const Http::HeaderMap* request_headers, + const Http::HeaderMap*, + std::vector& statuses) const { + if (request_headers != nullptr) { + statuses[my_index_] = Http::HeaderUtility::matchHeaders(*request_headers, headers_to_match_); + } + + return statuses[my_index_]; +} + +HttpResponseMatcher::HttpResponseMatcher( + const envoy::service::tap::v2alpha::HttpResponseMatch& config, + const std::vector& matchers) + : Matcher(matchers) { + for (const auto& header_match : config.headers()) { + headers_to_match_.emplace_back(header_match); + } +} + +bool HttpResponseMatcher::updateMatchStatus(const Http::HeaderMap*, + const Http::HeaderMap* response_headers, + std::vector& statuses) const { + if (response_headers != nullptr) { + statuses[my_index_] = Http::HeaderUtility::matchHeaders(*response_headers, headers_to_match_); + } + + return statuses[my_index_]; +} + +} // namespace Tap +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/common/tap/tap_matcher.h b/source/extensions/common/tap/tap_matcher.h new file mode 100644 index 000000000000..e01ff292316f --- /dev/null +++ b/source/extensions/common/tap/tap_matcher.h @@ -0,0 +1,174 @@ +#pragma once + +#include "envoy/service/tap/v2alpha/common.pb.h" + +#include "common/http/header_utility.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Tap { + +class Matcher; +using MatcherPtr = std::unique_ptr; + +/** + * Base class for all tap matchers. + * + * A high level note on the design of tap matching which is different from other matching in Envoy + * due to a requirement to support streaming matching (match as new data arrives versus + * calculating the match given all available data at once). + * - The matching system is composed of a constant matching configuration. This is essentially + * a tree of matchers given logical AND, OR, NOT, etc. + * - A per-stream/request matching status must be kept in order to compute interim match status. + * - In order to make this computationally efficient, the matching tree is kept in a vector, with + * all references to other matchers implemented using an index into the vector. The vector is + * effectively a flattened N-ary tree. + * - The previous point allows the creation of a per-stream/request vector of booleans of the same + * size as the matcher vector. Then, when match status is updated given new information, the + * vector of booleans can be easily updated using the same indexes as in the constant match + * configuration. + * - Finally, a matches() function can be trivially implemented by looking in the status vector at + * the index position that the current matcher is located in. + */ +class Matcher { +public: + virtual ~Matcher() = default; + + /** + * @return the matcher's index in the match tree vector (see above). + */ + size_t index() { return my_index_; } + + /** + * Update match status given new information. + * @param request_headers supplies the request headers, if available. + * @param response_headers supplies the response headers, if available. + * @param statuses supplies the per-stream-request match status vector which must be the same + * size as the match tree vector (see above). + */ + virtual bool updateMatchStatus(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + std::vector& statuses) const PURE; + + /** + * @return whether given currently available information, the matcher matches. + * @param statuses supplies the per-stream-request match status vector which must be the same + * size as the match tree vector (see above). + */ + bool matches(const std::vector& statuses) const { return statuses[my_index_]; } + +protected: + /** + * Base class constructor for a matcher. + * @param matchers supplies the match tree vector being built. + */ + Matcher(const std::vector& matchers) + // NOTE: This code assumes that the index for the matcher being constructed has already been + // allocated, which is why my_index_ is set to size() - 1. See buildMatcher() in + // tap_matcher.cc. + : my_index_(matchers.size() - 1) {} + + const size_t my_index_; +}; + +/** + * Factory method to build a matcher given a match config. Calling this function may end + * up recursively building many matchers, which will all be added to the passed in vector + * of matchers. See the comments in tap.h for the general structure of how tap matchers work. + */ +void buildMatcher(const envoy::service::tap::v2alpha::MatchPredicate& match_config, + std::vector& matchers); + +/** + * Matcher for implementing set logic. + */ +class SetLogicMatcher : public Matcher { +public: + enum class Type { And, Or }; + + SetLogicMatcher(const envoy::service::tap::v2alpha::MatchPredicate::MatchSet& configs, + std::vector& matchers, Type type); + + // Extensions::Common::Tap::Matcher + bool updateMatchStatus(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + std::vector& statuses) const override; + +private: + std::vector& matchers_; + std::vector indexes_; + const Type type_; +}; + +/** + * Not matcher. + */ +class NotMatcher : public Matcher { +public: + NotMatcher(const envoy::service::tap::v2alpha::MatchPredicate& config, + std::vector& matchers); + + // Extensions::Common::Tap::Matcher + bool updateMatchStatus(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + std::vector& statuses) const override; + +private: + std::vector& matchers_; + const size_t not_index_; +}; + +/** + * Any matcher (always matches). + */ +class AnyMatcher : public Matcher { +public: + AnyMatcher(std::vector& matchers) : Matcher(matchers) {} + + // Extensions::Common::Tap::Matcher + bool updateMatchStatus(const Http::HeaderMap*, const Http::HeaderMap*, + std::vector& statuses) const override { + statuses[my_index_] = true; + return true; + } +}; + +/** + * HTTP request matcher. + */ +class HttpRequestMatcher : public Matcher { +public: + HttpRequestMatcher(const envoy::service::tap::v2alpha::HttpRequestMatch& config, + const std::vector& matchers); + + // Extensions::Common::Tap::Matcher + bool updateMatchStatus(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + std::vector& statuses) const override; + +private: + std::vector headers_to_match_; +}; + +/** + * HTTP response matcher. + */ +class HttpResponseMatcher : public Matcher { +public: + HttpResponseMatcher(const envoy::service::tap::v2alpha::HttpResponseMatch& config, + const std::vector& matchers); + + // Extensions::Common::Tap::Matcher + bool updateMatchStatus(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers, + std::vector& statuses) const override; + +private: + std::vector headers_to_match_; +}; + +} // namespace Tap +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index 8685668310eb..1c00a1c0440e 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -42,6 +42,7 @@ EXTENSIONS = { "envoy.filters.http.rbac": "//source/extensions/filters/http/rbac:config", "envoy.filters.http.router": "//source/extensions/filters/http/router:config", "envoy.filters.http.squash": "//source/extensions/filters/http/squash:config", + "envoy.filters.http.tap": "//source/extensions/filters/http/tap:config", # # Listener filters diff --git a/source/extensions/filters/http/tap/BUILD b/source/extensions/filters/http/tap/BUILD new file mode 100644 index 000000000000..d44f143b0dfd --- /dev/null +++ b/source/extensions/filters/http/tap/BUILD @@ -0,0 +1,60 @@ +licenses(["notice"]) # Apache 2 + +# L7 HTTP Tap filter +# Public docs: docs/root/configuration/http_filters/tap_filter.rst + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "tap_config_interface", + hdrs = ["tap_config.h"], + deps = [ + "//include/envoy/http:header_map_interface", + "//source/extensions/common/tap:tap_interface", + "@envoy_api//envoy/service/tap/v2alpha:common_cc", + ], +) + +envoy_cc_library( + name = "tap_config_impl", + srcs = ["tap_config_impl.cc"], + hdrs = ["tap_config_impl.h"], + deps = [ + ":tap_config_interface", + "//source/extensions/common/tap:tap_config_base", + "@envoy_api//envoy/data/tap/v2alpha:http_cc", + ], +) + +envoy_cc_library( + name = "tap_filter_lib", + srcs = ["tap_filter.cc"], + hdrs = ["tap_filter.h"], + deps = [ + ":tap_config_interface", + "//include/envoy/http:filter_interface", + "//include/envoy/thread_local:thread_local_interface", + "//source/extensions/common/tap:admin", + "@envoy_api//envoy/config/filter/http/tap/v2alpha:tap_cc", + ], +) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":tap_config_impl", + ":tap_filter_lib", + "//include/envoy/registry", + "//source/extensions/filters/http:well_known_names", + "//source/extensions/filters/http/common:factory_base_lib", + "@envoy_api//envoy/config/filter/http/tap/v2alpha:tap_cc", + ], +) diff --git a/source/extensions/filters/http/tap/config.cc b/source/extensions/filters/http/tap/config.cc new file mode 100644 index 000000000000..746d26e5854b --- /dev/null +++ b/source/extensions/filters/http/tap/config.cc @@ -0,0 +1,45 @@ +#include "extensions/filters/http/tap/config.h" + +#include "envoy/registry/registry.h" + +#include "extensions/filters/http/tap/tap_config_impl.h" +#include "extensions/filters/http/tap/tap_filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TapFilter { + +class HttpTapConfigFactoryImpl : public HttpTapConfigFactory { +public: + HttpTapConfigSharedPtr + createHttpConfigFromProto(envoy::service::tap::v2alpha::TapConfig&& proto_config, + Extensions::Common::Tap::Sink* admin_streamer) override { + return std::make_shared(std::move(proto_config), admin_streamer); + } +}; + +Http::FilterFactoryCb TapFilterFactory::createFilterFactoryFromProtoTyped( + const envoy::config::filter::http::tap::v2alpha::Tap& proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) { + FilterConfigSharedPtr filter_config(new FilterConfigImpl( + proto_config, stats_prefix, std::make_unique(), context.scope(), + context.admin(), context.singletonManager(), context.threadLocal(), context.dispatcher())); + return [filter_config](Http::FilterChainFactoryCallbacks& callbacks) -> void { + auto filter = std::make_shared(filter_config); + callbacks.addStreamFilter(filter); + callbacks.addAccessLogHandler(filter); + }; +} + +/** + * Static registration for the tap filter. @see RegisterFactory. + */ +static Registry::RegisterFactory + register_; + +} // namespace TapFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/tap/config.h b/source/extensions/filters/http/tap/config.h new file mode 100644 index 000000000000..1819217ba3fc --- /dev/null +++ b/source/extensions/filters/http/tap/config.h @@ -0,0 +1,31 @@ +#pragma once + +#include "envoy/config/filter/http/tap/v2alpha/tap.pb.h" +#include "envoy/config/filter/http/tap/v2alpha/tap.pb.validate.h" + +#include "extensions/filters/http/common/factory_base.h" +#include "extensions/filters/http/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TapFilter { + +/** + * Config registration for the tap filter. + */ +class TapFilterFactory + : public Common::FactoryBase { +public: + TapFilterFactory() : FactoryBase(HttpFilterNames::get().Tap) {} + +private: + Http::FilterFactoryCb createFilterFactoryFromProtoTyped( + const envoy::config::filter::http::tap::v2alpha::Tap& proto_config, + const std::string& stats_prefix, Server::Configuration::FactoryContext& context) override; +}; + +} // namespace TapFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/tap/tap_config.h b/source/extensions/filters/http/tap/tap_config.h new file mode 100644 index 000000000000..7270a7b935a1 --- /dev/null +++ b/source/extensions/filters/http/tap/tap_config.h @@ -0,0 +1,78 @@ +#pragma once + +#include "envoy/common/pure.h" +#include "envoy/http/header_map.h" +#include "envoy/service/tap/v2alpha/common.pb.h" + +#include "extensions/common/tap/tap.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TapFilter { + +/** + * Per-request/stream HTTP tap implementation. Abstractly handles all request lifecycle events in + * order to tap if the configuration matches. + */ +class HttpPerRequestTapper { +public: + virtual ~HttpPerRequestTapper() = default; + + /** + * Called when request headers are received. + */ + virtual void onRequestHeaders(const Http::HeaderMap& headers) PURE; + + /** + * Called when response headers are received. + */ + virtual void onResponseHeaders(const Http::HeaderMap& headers) PURE; + + /** + * Called when the request is being destroyed and is being logged. + * @return whether the request was tapped or not. + */ + virtual bool onDestroyLog(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers) PURE; +}; + +using HttpPerRequestTapperPtr = std::unique_ptr; + +/** + * Abstract HTTP tap configuration. + */ +class HttpTapConfig { +public: + virtual ~HttpTapConfig() = default; + + /** + * @return a new per-request HTTP tapper which is used to handle tapping of a discrete request. + */ + virtual HttpPerRequestTapperPtr createPerRequestTapper() PURE; +}; + +using HttpTapConfigSharedPtr = std::shared_ptr; + +/** + * Configuration factory for the HTTP tap filter. + */ +class HttpTapConfigFactory { +public: + virtual ~HttpTapConfigFactory() = default; + + /** + * @return a new configuration given a raw tap service config proto. See + * Extensions::Common::Tap::ExtensionConfig::newTapConfig() for param info. + */ + virtual HttpTapConfigSharedPtr + createHttpConfigFromProto(envoy::service::tap::v2alpha::TapConfig&& proto_config, + Extensions::Common::Tap::Sink* admin_streamer) PURE; +}; + +using HttpTapConfigFactoryPtr = std::unique_ptr; + +} // namespace TapFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/tap/tap_config_impl.cc b/source/extensions/filters/http/tap/tap_config_impl.cc new file mode 100644 index 000000000000..9732b776559e --- /dev/null +++ b/source/extensions/filters/http/tap/tap_config_impl.cc @@ -0,0 +1,61 @@ +#include "extensions/filters/http/tap/tap_config_impl.h" + +#include "envoy/data/tap/v2alpha/http.pb.h" + +#include "common/common/assert.h" +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TapFilter { + +HttpTapConfigImpl::HttpTapConfigImpl(envoy::service::tap::v2alpha::TapConfig&& proto_config, + Common::Tap::Sink* admin_streamer) + : Extensions::Common::Tap::TapConfigBaseImpl(std::move(proto_config), admin_streamer) {} + +HttpPerRequestTapperPtr HttpTapConfigImpl::createPerRequestTapper() { + return std::make_unique(shared_from_this()); +} + +void HttpPerRequestTapperImpl::onRequestHeaders(const Http::HeaderMap& headers) { + config_->rootMatcher().updateMatchStatus(&headers, nullptr, statuses_); +} + +void HttpPerRequestTapperImpl::onResponseHeaders(const Http::HeaderMap& headers) { + config_->rootMatcher().updateMatchStatus(nullptr, &headers, statuses_); +} + +namespace { +Http::HeaderMap::Iterate fillHeaderList(const Http::HeaderEntry& header, void* context) { + Protobuf::RepeatedPtrField& header_list = + *reinterpret_cast*>(context); + auto& new_header = *header_list.Add(); + new_header.set_key(header.key().c_str()); + new_header.set_value(header.value().c_str()); + return Http::HeaderMap::Iterate::Continue; +} +} // namespace + +bool HttpPerRequestTapperImpl::onDestroyLog(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers) { + if (!config_->rootMatcher().matches(statuses_)) { + return false; + } + + auto trace = std::make_shared(); + auto& http_trace = *trace->mutable_http_buffered_trace(); + request_headers->iterate(fillHeaderList, http_trace.mutable_request_headers()); + if (response_headers != nullptr) { + response_headers->iterate(fillHeaderList, http_trace.mutable_response_headers()); + } + + ENVOY_LOG(debug, "submitting buffered trace sink"); + config_->sink().submitBufferedTrace(trace); + return true; +} + +} // namespace TapFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/tap/tap_config_impl.h b/source/extensions/filters/http/tap/tap_config_impl.h new file mode 100644 index 000000000000..b8f18e8fed96 --- /dev/null +++ b/source/extensions/filters/http/tap/tap_config_impl.h @@ -0,0 +1,47 @@ +#pragma once + +#include "envoy/http/header_map.h" + +#include "common/common/logger.h" + +#include "extensions/common/tap/tap_config_base.h" +#include "extensions/filters/http/tap/tap_config.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TapFilter { + +class HttpTapConfigImpl : public Extensions::Common::Tap::TapConfigBaseImpl, + public HttpTapConfig, + public std::enable_shared_from_this { +public: + HttpTapConfigImpl(envoy::service::tap::v2alpha::TapConfig&& proto_config, + Extensions::Common::Tap::Sink* admin_streamer); + + // TapFilter::HttpTapConfig + HttpPerRequestTapperPtr createPerRequestTapper() override; +}; + +using HttpTapConfigImplSharedPtr = std::shared_ptr; + +class HttpPerRequestTapperImpl : public HttpPerRequestTapper, Logger::Loggable { +public: + HttpPerRequestTapperImpl(HttpTapConfigImplSharedPtr config) + : config_(std::move(config)), statuses_(config_->numMatchers()) {} + + // TapFilter::HttpPerRequestTapper + void onRequestHeaders(const Http::HeaderMap& headers) override; + void onResponseHeaders(const Http::HeaderMap& headers) override; + bool onDestroyLog(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers) override; + +private: + HttpTapConfigImplSharedPtr config_; + std::vector statuses_; +}; + +} // namespace TapFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/tap/tap_filter.cc b/source/extensions/filters/http/tap/tap_filter.cc new file mode 100644 index 000000000000..318d9672120e --- /dev/null +++ b/source/extensions/filters/http/tap/tap_filter.cc @@ -0,0 +1,89 @@ +#include "extensions/filters/http/tap/tap_filter.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TapFilter { + +FilterConfigImpl::FilterConfigImpl(envoy::config::filter::http::tap::v2alpha::Tap proto_config, + const std::string& stats_prefix, + HttpTapConfigFactoryPtr&& config_factory, Stats::Scope& scope, + Server::Admin& admin, Singleton::Manager& singleton_manager, + ThreadLocal::SlotAllocator& tls, + Event::Dispatcher& main_thread_dispatcher) + : proto_config_(std::move(proto_config)), stats_(Filter::generateStats(stats_prefix, scope)), + config_factory_(std::move(config_factory)), tls_slot_(tls.allocateSlot()) { + + // TODO(mattklein123): Admin is the only supported config type currently. + ASSERT(proto_config_.has_admin_config()); + + admin_handler_ = Extensions::Common::Tap::AdminHandler::getSingleton(admin, singleton_manager, + main_thread_dispatcher); + admin_handler_->registerConfig(*this, proto_config_.admin_config().config_id()); + ENVOY_LOG(debug, "initializing tap filter with admin endpoint (config_id={})", + proto_config_.admin_config().config_id()); + + tls_slot_->set([](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(); + }); +} + +FilterConfigImpl::~FilterConfigImpl() { + if (admin_handler_) { + admin_handler_->unregisterConfig(*this); + } +} + +HttpTapConfigSharedPtr FilterConfigImpl::currentConfig() { + return tls_slot_->getTyped().config_; +} + +const std::string& FilterConfigImpl::adminId() { + ASSERT(proto_config_.has_admin_config()); + return proto_config_.admin_config().config_id(); +} + +void FilterConfigImpl::clearTapConfig() { + tls_slot_->runOnAllThreads([this] { tls_slot_->getTyped().config_ = nullptr; }); +} + +void FilterConfigImpl::newTapConfig(envoy::service::tap::v2alpha::TapConfig&& proto_config, + Common::Tap::Sink* admin_streamer) { + HttpTapConfigSharedPtr new_config = + config_factory_->createHttpConfigFromProto(std::move(proto_config), admin_streamer); + tls_slot_->runOnAllThreads( + [this, new_config] { tls_slot_->getTyped().config_ = new_config; }); +} + +FilterStats Filter::generateStats(const std::string& prefix, Stats::Scope& scope) { + // TODO(mattklein123): Consider whether we want to additionally namespace the stats on the + // filter's configured opaque ID. + std::string final_prefix = prefix + "tap."; + return {ALL_TAP_FILTER_STATS(POOL_COUNTER_PREFIX(scope, final_prefix))}; +} + +Http::FilterHeadersStatus Filter::decodeHeaders(Http::HeaderMap& headers, bool) { + if (tapper_ != nullptr) { + tapper_->onRequestHeaders(headers); + } + return Http::FilterHeadersStatus::Continue; +} + +Http::FilterHeadersStatus Filter::encodeHeaders(Http::HeaderMap& headers, bool) { + if (tapper_ != nullptr) { + tapper_->onResponseHeaders(headers); + } + return Http::FilterHeadersStatus::Continue; +} + +void Filter::log(const Http::HeaderMap* request_headers, const Http::HeaderMap* response_headers, + const Http::HeaderMap*, const StreamInfo::StreamInfo&) { + if (tapper_ != nullptr && tapper_->onDestroyLog(request_headers, response_headers)) { + config_->stats().rq_tapped_.inc(); + } +} + +} // namespace TapFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/tap/tap_filter.h b/source/extensions/filters/http/tap/tap_filter.h new file mode 100644 index 000000000000..144e601fcaff --- /dev/null +++ b/source/extensions/filters/http/tap/tap_filter.h @@ -0,0 +1,142 @@ +#pragma once + +#include "envoy/config/filter/http/tap/v2alpha/tap.pb.h" +#include "envoy/http/filter.h" +#include "envoy/service/tap/v2alpha/common.pb.h" +#include "envoy/stats/scope.h" +#include "envoy/stats/stats_macros.h" +#include "envoy/thread_local/thread_local.h" + +#include "extensions/common/tap/admin.h" +#include "extensions/filters/http/tap/tap_config.h" + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TapFilter { + +/** + * All stats for the tap filter. @see stats_macros.h + */ +// clang-format off +#define ALL_TAP_FILTER_STATS(COUNTER) \ + COUNTER(rq_tapped) +// clang-format on + +/** + * Wrapper struct for tap filter stats. @see stats_macros.h + */ +struct FilterStats { + ALL_TAP_FILTER_STATS(GENERATE_COUNTER_STRUCT) +}; + +/** + * Abstract filter configuration. + */ +class FilterConfig { +public: + virtual ~FilterConfig() = default; + + /** + * @return the current tap configuration if there is one. + */ + virtual HttpTapConfigSharedPtr currentConfig() PURE; + + /** + * @return the filter stats. + */ + virtual FilterStats& stats() PURE; +}; + +using FilterConfigSharedPtr = std::shared_ptr; + +/** + * Configuration for the tap filter. + */ +class FilterConfigImpl : public FilterConfig, + public Extensions::Common::Tap::ExtensionConfig, + Logger::Loggable { +public: + FilterConfigImpl(envoy::config::filter::http::tap::v2alpha::Tap proto_config, + const std::string& stats_prefix, HttpTapConfigFactoryPtr&& config_factory, + Stats::Scope& scope, Server::Admin& admin, Singleton::Manager& singleton_manager, + ThreadLocal::SlotAllocator& tls, Event::Dispatcher& main_thread_dispatcher); + ~FilterConfigImpl() override; + + // FilterConfig + HttpTapConfigSharedPtr currentConfig() override; + FilterStats& stats() override { return stats_; } + + // Extensions::Common::Tap::ExtensionConfig + void clearTapConfig() override; + const std::string& adminId() override; + void newTapConfig(envoy::service::tap::v2alpha::TapConfig&& proto_config, + Extensions::Common::Tap::Sink* admin_streamer) override; + +private: + struct TlsFilterConfig : public ThreadLocal::ThreadLocalObject { + HttpTapConfigSharedPtr config_; + }; + + const envoy::config::filter::http::tap::v2alpha::Tap proto_config_; + FilterStats stats_; + HttpTapConfigFactoryPtr config_factory_; + ThreadLocal::SlotPtr tls_slot_; + Extensions::Common::Tap::AdminHandlerSharedPtr admin_handler_; +}; + +/** + * HTTP tap filter. + */ +class Filter : public Http::StreamFilter, public AccessLog::Instance { +public: + Filter(FilterConfigSharedPtr config) + : config_(std::move(config)), + tapper_(config_->currentConfig() ? config_->currentConfig()->createPerRequestTapper() + : nullptr) {} + + static FilterStats generateStats(const std::string& prefix, Stats::Scope& scope); + + // Http::StreamFilterBase + void onDestroy() override {} + + // Http::StreamDecoderFilter + Http::FilterHeadersStatus decodeHeaders(Http::HeaderMap& headers, bool end_stream) override; + Http::FilterDataStatus decodeData(Buffer::Instance&, bool) override { + return Http::FilterDataStatus::Continue; + } + Http::FilterTrailersStatus decodeTrailers(Http::HeaderMap&) override { + return Http::FilterTrailersStatus::Continue; + } + void setDecoderFilterCallbacks(Http::StreamDecoderFilterCallbacks&) override {} + + // Http::StreamEncoderFilter + Http::FilterHeadersStatus encode100ContinueHeaders(Http::HeaderMap&) override { + return Http::FilterHeadersStatus::Continue; + } + Http::FilterHeadersStatus encodeHeaders(Http::HeaderMap& headers, bool end_stream) override; + Http::FilterDataStatus encodeData(Buffer::Instance&, bool) override { + return Http::FilterDataStatus::Continue; + } + Http::FilterTrailersStatus encodeTrailers(Http::HeaderMap&) override { + return Http::FilterTrailersStatus::Continue; + } + Http::FilterMetadataStatus encodeMetadata(Http::MetadataMap&) override { + return Http::FilterMetadataStatus::Continue; + } + void setEncoderFilterCallbacks(Http::StreamEncoderFilterCallbacks&) override {} + + // AccessLog::Instance + void log(const Http::HeaderMap* request_headers, const Http::HeaderMap* response_headers, + const Http::HeaderMap* response_trailers, + const StreamInfo::StreamInfo& stream_info) override; + +private: + FilterConfigSharedPtr config_; + HttpPerRequestTapperPtr tapper_; +}; + +} // namespace TapFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/filters/http/well_known_names.h b/source/extensions/filters/http/well_known_names.h index cda225da77a1..d3585225d752 100644 --- a/source/extensions/filters/http/well_known_names.h +++ b/source/extensions/filters/http/well_known_names.h @@ -50,6 +50,8 @@ class HttpFilterNameValues { const std::string JwtAuthn = "envoy.filters.http.jwt_authn"; // Header to metadata filter const std::string HeaderToMetadata = "envoy.filters.http.header_to_metadata"; + // Tap filter + const std::string Tap = "envoy.filters.http.tap"; // Converts names from v1 to v2 const Config::V1Converter v1_converter_; diff --git a/test/extensions/common/tap/BUILD b/test/extensions/common/tap/BUILD new file mode 100644 index 000000000000..a60a8f29f2a3 --- /dev/null +++ b/test/extensions/common/tap/BUILD @@ -0,0 +1,26 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +envoy_package() + +envoy_cc_test( + name = "admin_test", + srcs = ["admin_test.cc"], + deps = [ + "//source/extensions/common/tap:admin", + "//test/mocks/server:server_mocks", + ], +) + +envoy_cc_test( + name = "tap_matcher_test", + srcs = ["tap_matcher_test.cc"], + deps = [ + "//source/extensions/common/tap:tap_matcher", + ], +) diff --git a/test/extensions/common/tap/admin_test.cc b/test/extensions/common/tap/admin_test.cc new file mode 100644 index 000000000000..2440c844afc8 --- /dev/null +++ b/test/extensions/common/tap/admin_test.cc @@ -0,0 +1,99 @@ +#include "extensions/common/tap/admin.h" + +#include "test/mocks/server/mocks.h" + +#include "gtest/gtest.h" + +using testing::_; +using testing::Return; +using testing::SaveArg; + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Tap { +namespace { + +class MockExtensionConfig : public ExtensionConfig { +public: + MOCK_METHOD0(adminId, const std::string&()); + MOCK_METHOD0(clearTapConfig, void()); + MOCK_METHOD2(newTapConfig, + void(envoy::service::tap::v2alpha::TapConfig&& proto_config, Sink* admin_streamer)); +}; + +class AdminHandlerTest : public testing::Test { +public: + AdminHandlerTest() { + EXPECT_CALL(admin_, addHandler("/tap", "tap filter control", _, true, true)) + .WillOnce(DoAll(SaveArg<2>(&cb_), Return(true))); + handler_ = std::make_unique(admin_, main_thread_dispatcher_); + } + + ~AdminHandlerTest() { EXPECT_CALL(admin_, removeHandler("/tap")).WillOnce(Return(true)); } + + Server::MockAdmin admin_; + Event::MockDispatcher main_thread_dispatcher_; + std::unique_ptr handler_; + Server::Admin::HandlerCb cb_; + Http::TestHeaderMapImpl response_headers_; + Buffer::OwnedImpl response_; + Server::MockAdminStream admin_stream_; + + const std::string admin_request_yaml_ = + R"EOF( +config_id: test_config_id +tap_config: + match_config: + any_match: true + output_config: + sinks: + - streaming_admin: {} +)EOF"; +}; + +// Request with no config body. +TEST_F(AdminHandlerTest, NoBody) { + EXPECT_CALL(admin_stream_, getRequestBody()); + EXPECT_EQ(Http::Code::BadRequest, cb_("/tap", response_headers_, response_, admin_stream_)); + EXPECT_EQ("/tap requires a JSON/YAML body", response_.toString()); +} + +// Request with a config body that doesn't parse/verify. +TEST_F(AdminHandlerTest, BadBody) { + Buffer::OwnedImpl bad_body("hello"); + EXPECT_CALL(admin_stream_, getRequestBody()).WillRepeatedly(Return(&bad_body)); + EXPECT_EQ(Http::Code::BadRequest, cb_("/tap", response_headers_, response_, admin_stream_)); + EXPECT_EQ("Unable to convert YAML as JSON: hello", response_.toString()); +} + +// Request that references an unknown config ID. +TEST_F(AdminHandlerTest, UnknownConfigId) { + Buffer::OwnedImpl body(admin_request_yaml_); + EXPECT_CALL(admin_stream_, getRequestBody()).WillRepeatedly(Return(&body)); + EXPECT_EQ(Http::Code::BadRequest, cb_("/tap", response_headers_, response_, admin_stream_)); + EXPECT_EQ("Unknown config id 'test_config_id'. No extension has registered with this id.", + response_.toString()); +} + +// Request while there is already an active tap session. +TEST_F(AdminHandlerTest, RequestTapWhileAttached) { + MockExtensionConfig extension_config; + handler_->registerConfig(extension_config, "test_config_id"); + + Buffer::OwnedImpl body(admin_request_yaml_); + EXPECT_CALL(admin_stream_, getRequestBody()).WillRepeatedly(Return(&body)); + EXPECT_CALL(extension_config, newTapConfig(_, handler_.get())); + EXPECT_CALL(admin_stream_, setEndStreamOnComplete(false)); + EXPECT_CALL(admin_stream_, addOnDestroyCallback(_)); + EXPECT_EQ(Http::Code::OK, cb_("/tap", response_headers_, response_, admin_stream_)); + + EXPECT_EQ(Http::Code::BadRequest, cb_("/tap", response_headers_, response_, admin_stream_)); + EXPECT_EQ("An attached /tap admin stream already exists. Detach it.", response_.toString()); +} + +} // namespace +} // namespace Tap +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/common/tap/tap_matcher_test.cc b/test/extensions/common/tap/tap_matcher_test.cc new file mode 100644 index 000000000000..b6798f0a0a76 --- /dev/null +++ b/test/extensions/common/tap/tap_matcher_test.cc @@ -0,0 +1,51 @@ +#include "common/protobuf/utility.h" + +#include "extensions/common/tap/tap_matcher.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace Extensions { +namespace Common { +namespace Tap { +namespace { + +class TapMatcherTest : public testing::Test { +public: + std::vector matchers_; + std::vector statuses_; + envoy::service::tap::v2alpha::MatchPredicate config_; +}; + +TEST_F(TapMatcherTest, Any) { + const std::string matcher_yaml = + R"EOF( +any_match: true +)EOF"; + + MessageUtil::loadFromYaml(matcher_yaml, config_); + buildMatcher(config_, matchers_); + EXPECT_EQ(1, matchers_.size()); + statuses_.resize(matchers_.size()); + EXPECT_TRUE(matchers_[0]->updateMatchStatus(nullptr, nullptr, statuses_)); +} + +TEST_F(TapMatcherTest, Not) { + const std::string matcher_yaml = + R"EOF( +not_match: + any_match: true +)EOF"; + + MessageUtil::loadFromYaml(matcher_yaml, config_); + buildMatcher(config_, matchers_); + EXPECT_EQ(2, matchers_.size()); + statuses_.resize(matchers_.size()); + EXPECT_FALSE(matchers_[0]->updateMatchStatus(nullptr, nullptr, statuses_)); +} + +} // namespace +} // namespace Tap +} // namespace Common +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/filters/http/tap/BUILD b/test/extensions/filters/http/tap/BUILD new file mode 100644 index 000000000000..a6e7f916d6c2 --- /dev/null +++ b/test/extensions/filters/http/tap/BUILD @@ -0,0 +1,33 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "tap_filter_test", + srcs = ["tap_filter_test.cc"], + extension_name = "envoy.filters.http.tap", + deps = [ + "//source/extensions/filters/http/tap:tap_filter_lib", + "//test/mocks/stream_info:stream_info_mocks", + "//test/test_common:utility_lib", + ], +) + +envoy_extension_cc_test( + name = "tap_filter_integration_test", + srcs = ["tap_filter_integration_test.cc"], + extension_name = "envoy.filters.http.tap", + deps = [ + "//source/extensions/filters/http/tap:config", + "//test/integration:http_integration_lib", + ], +) diff --git a/test/extensions/filters/http/tap/tap_filter_integration_test.cc b/test/extensions/filters/http/tap/tap_filter_integration_test.cc new file mode 100644 index 000000000000..0f6c0111efdc --- /dev/null +++ b/test/extensions/filters/http/tap/tap_filter_integration_test.cc @@ -0,0 +1,210 @@ +#include "envoy/data/tap/v2alpha/wrapper.pb.h" + +#include "test/integration/http_integration.h" + +#include "gtest/gtest.h" + +namespace Envoy { +namespace { + +class TapIntegrationTest : public HttpIntegrationTest, + public testing::TestWithParam { +public: + TapIntegrationTest() + // Note: This test must use HTTP/2 because of the lack of early close detection for + // HTTP/1 on OSX. In this test we close the admin /tap stream when we don't want any + // more data, and without immediate close detection we can't have a flake free test. + // Thus, we use HTTP/2 for everything here. + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, GetParam(), realTime()) {} + + void initializeFilter(const std::string& filter_config) { + config_helper_.addFilter(filter_config); + initialize(); + } + + const envoy::api::v2::core::HeaderValue* + findHeader(const std::string& key, + const Protobuf::RepeatedPtrField& headers) { + for (const auto& header : headers) { + if (header.key() == key) { + return &header; + } + } + + return nullptr; + } +}; + +INSTANTIATE_TEST_CASE_P(IpVersions, TapIntegrationTest, + testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), + TestUtility::ipTestParamsToString); + +// Verify a basic tap flow using the admin handler. +TEST_P(TapIntegrationTest, AdminBasicFlow) { + const std::string FILTER_CONFIG = + R"EOF( +name: envoy.filters.http.tap +config: + admin_config: + config_id: test_config_id +)EOF"; + + initializeFilter(FILTER_CONFIG); + + // Initial request/response with no tap. + codec_client_ = makeHttpConnection(makeClientConnection(lookupPort("http"))); + const Http::TestHeaderMapImpl request_headers_tap{{":method", "GET"}, + {":path", "/"}, + {":scheme", "http"}, + {":authority", "host"}, + {"foo", "bar"}}; + IntegrationStreamDecoderPtr decoder = codec_client_->makeHeaderOnlyRequest(request_headers_tap); + waitForNextUpstreamRequest(); + const Http::TestHeaderMapImpl response_headers_no_tap{{":status", "200"}}; + upstream_request_->encodeHeaders(response_headers_no_tap, true); + decoder->waitForEndStream(); + + const std::string admin_request_yaml = + R"EOF( +config_id: test_config_id +tap_config: + match_config: + or_match: + rules: + - http_request_match: + headers: + - name: foo + exact_match: bar + - http_response_match: + headers: + - name: bar + exact_match: baz + output_config: + sinks: + - streaming_admin: {} +)EOF"; + + // Setup a tap and disconnect it without any request/response. + IntegrationCodecClientPtr admin_client_ = + makeHttpConnection(makeClientConnection(lookupPort("admin"))); + const Http::TestHeaderMapImpl admin_request_headers{ + {":method", "POST"}, {":path", "/tap"}, {":scheme", "http"}, {":authority", "host"}}; + IntegrationStreamDecoderPtr admin_response = + admin_client_->makeRequestWithBody(admin_request_headers, admin_request_yaml); + admin_response->waitForHeaders(); + EXPECT_STREQ("200", admin_response->headers().Status()->value().c_str()); + EXPECT_FALSE(admin_response->complete()); + admin_client_->close(); + test_server_->waitForGaugeEq("http.admin.downstream_rq_active", 0); + + // Second request/response with no tap. + decoder = codec_client_->makeHeaderOnlyRequest(request_headers_tap); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(response_headers_no_tap, true); + decoder->waitForEndStream(); + + // Setup the tap again and leave it open. + admin_client_ = makeHttpConnection(makeClientConnection(lookupPort("admin"))); + admin_response = admin_client_->makeRequestWithBody(admin_request_headers, admin_request_yaml); + admin_response->waitForHeaders(); + EXPECT_STREQ("200", admin_response->headers().Status()->value().c_str()); + EXPECT_FALSE(admin_response->complete()); + + // Do a request which should tap, matching on request headers. + decoder = codec_client_->makeHeaderOnlyRequest(request_headers_tap); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(response_headers_no_tap, true); + decoder->waitForEndStream(); + + // Wait for the tap message. + admin_response->waitForBodyData(1); + envoy::data::tap::v2alpha::BufferedTraceWrapper trace; + MessageUtil::loadFromYaml(admin_response->body(), trace); + EXPECT_EQ(trace.http_buffered_trace().request_headers().size(), 8); + EXPECT_EQ(trace.http_buffered_trace().response_headers().size(), 5); + admin_response->clearBody(); + + // Do a request which should not tap. + const Http::TestHeaderMapImpl request_headers_no_tap{ + {":method", "GET"}, {":path", "/"}, {":scheme", "http"}, {":authority", "host"}}; + decoder = codec_client_->makeHeaderOnlyRequest(request_headers_no_tap); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(response_headers_no_tap, true); + decoder->waitForEndStream(); + + // Do a request which should tap, matching on response headers. + decoder = codec_client_->makeHeaderOnlyRequest(request_headers_no_tap); + waitForNextUpstreamRequest(); + const Http::TestHeaderMapImpl response_headers_tap{{":status", "200"}, {"bar", "baz"}}; + upstream_request_->encodeHeaders(response_headers_tap, true); + decoder->waitForEndStream(); + + // Wait for the tap message. + admin_response->waitForBodyData(1); + MessageUtil::loadFromYaml(admin_response->body(), trace); + EXPECT_EQ(trace.http_buffered_trace().request_headers().size(), 7); + EXPECT_EQ( + "http", + findHeader("x-forwarded-proto", trace.http_buffered_trace().request_headers())->value()); + EXPECT_EQ(trace.http_buffered_trace().response_headers().size(), 6); + EXPECT_NE(nullptr, findHeader("date", trace.http_buffered_trace().response_headers())); + EXPECT_EQ("baz", findHeader("bar", trace.http_buffered_trace().response_headers())->value()); + + admin_client_->close(); + test_server_->waitForGaugeEq("http.admin.downstream_rq_active", 0); + + // Now setup a tap that matches on logical AND. + const std::string admin_request_yaml2 = + R"EOF( +config_id: test_config_id +tap_config: + match_config: + and_match: + rules: + - http_request_match: + headers: + - name: foo + exact_match: bar + - http_response_match: + headers: + - name: bar + exact_match: baz + output_config: + sinks: + - streaming_admin: {} +)EOF"; + + admin_client_ = makeHttpConnection(makeClientConnection(lookupPort("admin"))); + admin_response = admin_client_->makeRequestWithBody(admin_request_headers, admin_request_yaml2); + admin_response->waitForHeaders(); + EXPECT_STREQ("200", admin_response->headers().Status()->value().c_str()); + EXPECT_FALSE(admin_response->complete()); + + // Do a request that matches, but the response does not match. No tap. + decoder = codec_client_->makeHeaderOnlyRequest(request_headers_tap); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(response_headers_no_tap, true); + decoder->waitForEndStream(); + + // Do a request that doesn't match, but the response does match. No tap. + decoder = codec_client_->makeHeaderOnlyRequest(request_headers_no_tap); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(response_headers_tap, true); + decoder->waitForEndStream(); + + // Do a request that matches and a response that matches. Should tap. + decoder = codec_client_->makeHeaderOnlyRequest(request_headers_tap); + waitForNextUpstreamRequest(); + upstream_request_->encodeHeaders(response_headers_tap, true); + decoder->waitForEndStream(); + + // Wait for the tap message. + admin_response->waitForBodyData(1); + MessageUtil::loadFromYaml(admin_response->body(), trace); + + admin_client_->close(); + EXPECT_EQ(3UL, test_server_->counter("http.config_test.tap.rq_tapped")->value()); +} + +} // namespace +} // namespace Envoy diff --git a/test/extensions/filters/http/tap/tap_filter_test.cc b/test/extensions/filters/http/tap/tap_filter_test.cc new file mode 100644 index 000000000000..cf9dce466354 --- /dev/null +++ b/test/extensions/filters/http/tap/tap_filter_test.cc @@ -0,0 +1,130 @@ +#include "extensions/filters/http/tap/tap_filter.h" + +#include "test/mocks/stream_info/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::InSequence; +using testing::Return; + +namespace Envoy { +namespace Extensions { +namespace HttpFilters { +namespace TapFilter { +namespace { + +class MockFilterConfig : public FilterConfig { +public: + MOCK_METHOD0(currentConfig, HttpTapConfigSharedPtr()); + FilterStats& stats() override { return stats_; } + + Stats::IsolatedStoreImpl stats_store_; + FilterStats stats_{Filter::generateStats("foo", stats_store_)}; +}; + +class MockHttpTapConfig : public HttpTapConfig { +public: + HttpPerRequestTapperPtr createPerRequestTapper() override { + return HttpPerRequestTapperPtr{createPerRequestTapper_()}; + } + + MOCK_METHOD0(createPerRequestTapper_, HttpPerRequestTapper*()); +}; + +class MockHttpPerRequestTapper : public HttpPerRequestTapper { +public: + MOCK_METHOD1(onRequestHeaders, void(const Http::HeaderMap& headers)); + MOCK_METHOD1(onResponseHeaders, void(const Http::HeaderMap& headers)); + MOCK_METHOD2(onDestroyLog, bool(const Http::HeaderMap* request_headers, + const Http::HeaderMap* response_headers)); +}; + +class TapFilterTest : public testing::Test { +public: + void setup(bool has_config) { + if (has_config) { + http_tap_config_ = std::make_shared(); + } + + EXPECT_CALL(*filter_config_, currentConfig()).WillRepeatedly(Return(http_tap_config_)); + if (has_config) { + http_per_request_tapper_ = new MockHttpPerRequestTapper(); + EXPECT_CALL(*http_tap_config_, createPerRequestTapper_()) + .WillOnce(Return(http_per_request_tapper_)); + } + + filter_ = std::make_unique(filter_config_); + } + + std::shared_ptr filter_config_{new MockFilterConfig()}; + std::shared_ptr http_tap_config_; + MockHttpPerRequestTapper* http_per_request_tapper_; + std::unique_ptr filter_; + StreamInfo::MockStreamInfo stream_info_; +}; + +// Verify filter functionality when there is no tap config. +TEST_F(TapFilterTest, NoConfig) { + InSequence s; + setup(false); + + Http::TestHeaderMapImpl request_headers; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + Buffer::OwnedImpl request_body; + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(request_body, false)); + Http::TestHeaderMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + + Http::TestHeaderMapImpl response_headers; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->encode100ContinueHeaders(response_headers)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers, false)); + Buffer::OwnedImpl response_body; + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(response_body, false)); + Http::TestHeaderMapImpl response_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers)); + Http::MetadataMap metadata; + EXPECT_EQ(Http::FilterMetadataStatus::Continue, filter_->encodeMetadata(metadata)); + + filter_->log(&request_headers, &response_headers, &response_trailers, stream_info_); +} + +// Verify filter functionality when there is a tap config. +TEST_F(TapFilterTest, Config) { + InSequence s; + setup(true); + + Http::TestHeaderMapImpl request_headers; + EXPECT_CALL(*http_per_request_tapper_, onRequestHeaders(_)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->decodeHeaders(request_headers, false)); + Buffer::OwnedImpl request_body; + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->decodeData(request_body, false)); + Http::TestHeaderMapImpl request_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->decodeTrailers(request_trailers)); + + Http::TestHeaderMapImpl response_headers; + EXPECT_EQ(Http::FilterHeadersStatus::Continue, + filter_->encode100ContinueHeaders(response_headers)); + EXPECT_CALL(*http_per_request_tapper_, onResponseHeaders(_)); + EXPECT_EQ(Http::FilterHeadersStatus::Continue, filter_->encodeHeaders(response_headers, false)); + Buffer::OwnedImpl response_body; + EXPECT_EQ(Http::FilterDataStatus::Continue, filter_->encodeData(response_body, false)); + Http::TestHeaderMapImpl response_trailers; + EXPECT_EQ(Http::FilterTrailersStatus::Continue, filter_->encodeTrailers(response_trailers)); + + EXPECT_CALL(*http_per_request_tapper_, onDestroyLog(&request_headers, &response_headers)) + .WillOnce(Return(true)); + filter_->log(&request_headers, &response_headers, &response_trailers, stream_info_); + EXPECT_EQ(1UL, filter_config_->stats_.rq_tapped_.value()); + + // Workaround InSequence/shared_ptr mock leak. + EXPECT_TRUE(testing::Mock::VerifyAndClearExpectations(http_tap_config_.get())); +} + +} // namespace +} // namespace TapFilter +} // namespace HttpFilters +} // namespace Extensions +} // namespace Envoy diff --git a/test/proto/helloworld.proto b/test/proto/helloworld.proto index 2372a603b49f..007991440cd0 100644 --- a/test/proto/helloworld.proto +++ b/test/proto/helloworld.proto @@ -30,7 +30,6 @@ syntax = "proto3"; option cc_generic_services = true; -option java_multiple_files = true; option java_package = "io.grpc.examples.helloworld"; option java_outer_classname = "HelloWorldProto";