forked from envoyproxy/envoy
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Thrift: Payload to metadata filter (envoyproxy#23409)
This filter is configured with request_rules that will be matched against requests. A field_selector of a rule represents the head of a linked list, each node of the linked list has a name for logging and an id for matching. The field_selector is tied to a payload field when the linked list corresponds to a downward path which rooted in the top-level of the request message structure. on_present is triggered when corresponding the payload is present. Otherwise, on_missing is triggered. This filter is designed to support payload passthrough. By performing payload to metadata filter can do deserialization once, and pass the metadata to other filters. This means that load balancing decisions, consumed from log and routing could all use payload information with a single parse. Also notably performing the parsing in payload passthrough buffer will mean deserialization once and not re-serializing, which is the most performant outcome. Risk Level: low Testing: unit Docs Changes: multiple rst Fixes envoyproxy#23322 Signed-off-by: kuochunghsu <[email protected]>
- Loading branch information
Showing
20 changed files
with
2,419 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 12 additions & 0 deletions
12
api/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3/BUILD
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| # DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py. | ||
|
|
||
| load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package") | ||
|
|
||
| licenses(["notice"]) # Apache 2 | ||
|
|
||
| api_proto_package( | ||
| deps = [ | ||
| "//envoy/type/matcher/v3:pkg", | ||
| "@com_github_cncf_udpa//udpa/annotations:pkg", | ||
| ], | ||
| ) |
100 changes: 100 additions & 0 deletions
100
...ons/filters/network/thrift_proxy/filters/payload_to_metadata/v3/payload_to_metadata.proto
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,100 @@ | ||
| syntax = "proto3"; | ||
|
|
||
| package envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3; | ||
|
|
||
| import "envoy/type/matcher/v3/regex.proto"; | ||
|
|
||
| import "udpa/annotations/status.proto"; | ||
| import "validate/validate.proto"; | ||
|
|
||
| option java_package = "io.envoyproxy.envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3"; | ||
| option java_outer_classname = "PayloadToMetadataProto"; | ||
| option java_multiple_files = true; | ||
| option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3;payload_to_metadatav3"; | ||
| option (udpa.annotations.file_status).package_version_status = ACTIVE; | ||
|
|
||
| // [#protodoc-title: Payload-To-Metadata Filter] | ||
| // | ||
| // The configuration for transforming payloads into metadata. This is useful | ||
| // for matching load balancer subsets, logging, etc. | ||
| // | ||
| // Payload to Metadata :ref:`configuration overview <config_thrift_filters_payload_to_metadata>`. | ||
| // [#extension: envoy.filters.thrift.payload_to_metadata] | ||
|
|
||
| message PayloadToMetadata { | ||
| enum ValueType { | ||
| STRING = 0; | ||
| NUMBER = 1; | ||
| } | ||
|
|
||
| // [#next-free-field: 6] | ||
| message KeyValuePair { | ||
| // The namespace — if this is empty, the filter's namespace will be used. | ||
| string metadata_namespace = 1; | ||
|
|
||
| // The key to use within the namespace. | ||
| string key = 2 [(validate.rules).string = {min_len: 1}]; | ||
|
|
||
| oneof value_type { | ||
| // The value to pair with the given key. | ||
| // | ||
| // When used for on_present case, if value is non-empty it'll be used instead | ||
| // of the field value. If both are empty, the field value is used as-is. | ||
| // | ||
| // When used for on_missing case, a non-empty value must be provided. | ||
| string value = 3; | ||
|
|
||
| // If present, the header's value will be matched and substituted with this. | ||
| // If there is no match or substitution, the field value is used as-is. | ||
| // | ||
| // This is only used for on_present. | ||
| type.matcher.v3.RegexMatchAndSubstitute regex_value_rewrite = 4; | ||
| } | ||
|
|
||
| // The value's type — defaults to string. | ||
| ValueType type = 5 [(validate.rules).enum = {defined_only: true}]; | ||
| } | ||
|
|
||
| // A Rule defines what metadata to apply when a field is present or missing. | ||
| // [#next-free-field: 6] | ||
| message Rule { | ||
| oneof match_specifier { | ||
| option (validate.required) = true; | ||
|
|
||
| // If specified, the route must exactly match the request method name. As a special case, | ||
| // an empty string matches any request method name. | ||
| string method_name = 1; | ||
|
|
||
| // If specified, the route must have the service name as the request method name prefix. | ||
| // As a special case, an empty string matches any service name. Only relevant when service | ||
| // multiplexing. | ||
| string service_name = 2; | ||
| } | ||
|
|
||
| // Specifies that a match will be performed on the value of a field. | ||
| FieldSelector field_selector = 3 [(validate.rules).message = {required: true}]; | ||
|
|
||
| // If the field is present, apply this metadata KeyValuePair. | ||
| KeyValuePair on_present = 4; | ||
|
|
||
| // If the field is missing, apply this metadata KeyValuePair. | ||
| // | ||
| // The value in the KeyValuePair must be set, since it'll be used in lieu | ||
| // of the missing field value. | ||
| KeyValuePair on_missing = 5; | ||
| } | ||
|
|
||
| message FieldSelector { | ||
| // field name to log | ||
| string name = 1 [(validate.rules).string = {min_len: 1}]; | ||
|
|
||
| // field id to match | ||
| int32 id = 2 [(validate.rules).int32 = {lte: 32767 gte: -32768}]; | ||
|
|
||
| // next node of the field selector | ||
| FieldSelector child = 3; | ||
| } | ||
|
|
||
| // The list of rules to apply to requests. | ||
| repeated Rule request_rules = 1 [(validate.rules).repeated = {min_items: 1}]; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
72 changes: 72 additions & 0 deletions
72
...oot/configuration/other_protocols/thrift_filters/_include/payload-to-metadata-filter.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| static_resources: | ||
| listeners: | ||
| - address: | ||
| socket_address: | ||
| address: 0.0.0.0 | ||
| port_value: 9090 | ||
| filter_chains: | ||
| - filters: | ||
| - name: envoy.filters.network.thrift_proxy | ||
| typed_config: | ||
| "@type": type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.v3.ThriftProxy | ||
| stat_prefix: ingress_thrift | ||
| route_config: | ||
| name: local_route | ||
| routes: | ||
| - match: | ||
| method_name: "" | ||
| route: | ||
| cluster: versioned-cluster | ||
| thrift_filters: | ||
| - name: envoy.filters.thrift.payload_to_metadata | ||
| typed_config: | ||
| "@type": type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata | ||
| request_rules: | ||
| - method_name: foo | ||
| field_selector: | ||
| name: info | ||
| id: 2 | ||
| child: | ||
| name: version | ||
| id: 1 | ||
| on_present: | ||
| metadata_namespace: envoy.lb | ||
| key: version | ||
| on_missing: | ||
| metadata_namespace: envoy.lb | ||
| key: default | ||
| value: 'unknown' | ||
| clusters: | ||
| - name: versioned-cluster | ||
| type: STRICT_DNS | ||
| lb_policy: ROUND_ROBIN | ||
| lb_subset_config: | ||
| fallback_policy: NO_FALLBACK | ||
| subset_selectors: | ||
| - keys: | ||
| - default | ||
| - keys: | ||
| - version | ||
| load_assignment: | ||
| cluster_name: versioned-cluster | ||
| endpoints: | ||
| - lb_endpoints: | ||
| - endpoint: | ||
| address: | ||
| socket_address: | ||
| address: 127.0.0.1 | ||
| port_value: 19090 | ||
| metadata: | ||
| filter_metadata: | ||
| envoy.lb: | ||
| default: "true" | ||
| - lb_endpoints: | ||
| - endpoint: | ||
| address: | ||
| socket_address: | ||
| address: 127.0.0.1 | ||
| port_value: 19091 | ||
| metadata: | ||
| filter_metadata: | ||
| envoy.lb: | ||
| version: "1.0" |
11 changes: 11 additions & 0 deletions
11
docs/root/configuration/other_protocols/thrift_filters/_include/request.proto
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| syntax = "proto3"; | ||
|
|
||
| package request; | ||
|
|
||
| message Request { | ||
| message Info { | ||
| string version = 1; | ||
| } | ||
| string data = 1; | ||
| Info info = 2; | ||
| } |
107 changes: 107 additions & 0 deletions
107
...oot/configuration/other_protocols/thrift_filters/payload_to_metadata_filter.rst
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| .. _config_thrift_filters_payload_to_metadata: | ||
|
|
||
| Envoy Payload-To-Metadata Filter | ||
| ================================ | ||
| * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata``. | ||
| * :ref:`v3 API reference <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata>` | ||
|
|
||
| A typical use case for this filter is to dynamically match a specified payload field of requests | ||
| with load balancer subsets. For this, a given payload field's value would be extracted and attached | ||
| to the request as dynamic metadata which would then be used to match a subset of endpoints. | ||
|
|
||
| We already have :ref:`header-To-metadata filter <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.filters.header_to_metadata.v3.HeaderToMetadata>` | ||
| to achieve the similar goal. However, we have two reasons for introducing new :ref:`payload-To-metadata filter | ||
| <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata>`: | ||
|
|
||
| 1. Transports like framed transport don't support THeaders, which is unable to use :ref:`Header-To-Metadata filter | ||
| <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.filters.header_to_metadata.v3.HeaderToMetadata>`. | ||
|
|
||
| 2. Directly referring to payload field stops envoy relying on that the downstream service always copies the field | ||
| to the THeader correctly and guarantees single truth of source. | ||
|
|
||
| This filter is configured with :ref:`request_rules | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.request_rules>` | ||
| that will be matched against requests. A | ||
| :ref:`field_selector | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule.field_selector>` | ||
| of a :ref:`rule | ||
| <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule>` | ||
| represents the head of a linked list, each node of the linked list has a :ref:`name | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.FieldSelector.name>` | ||
| for logging and an :ref:`id | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.FieldSelector.id>` | ||
| for matching. The :ref:`field_selector | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule.field_selector>` | ||
| is tied to a payload field when the linked list corresponds to a downward path which rooted in the top-level of the | ||
| request message structure. :ref:`on_present | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule.on_present>` | ||
| is triggered when corresponding the payload is present. Otherwise, :ref:`on_missing | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule.on_missing>` | ||
| is triggered. | ||
|
|
||
| Note that if the corresponding payload for a :ref:`rule | ||
| <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule>` | ||
| is present but :ref:`on_present | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule.on_present>` | ||
| is missing, no metadata is added for this :ref:`rule | ||
| <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule>`. | ||
| . If the corresponding payload for a :ref:`rule | ||
| <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule>` | ||
| is an empty string, neither :ref:`on_present | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule.on_present>` | ||
| nor :ref:`on_missing | ||
| <envoy_v3_api_field_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule.on_missing>` | ||
| is triggered. i.e., no metadata is added for this :ref:`rule | ||
| <envoy_v3_api_msg_extensions.filters.network.thrift_proxy.filters.payload_to_metadata.v3.PayloadToMetadata.Rule>`. | ||
|
|
||
| Currently payload to metadata filter doesn't support container type payload, i.e., list, set and map. | ||
|
|
||
| We limit the size of a single metadata value which is added by this filter to 1024 bytes. | ||
|
|
||
| This filter is designed to support payload passthrough. Performing payload to metadata filter | ||
| can do deserialization once, and pass the metadata to other filters. This means that load balancing | ||
| decisions, consumed from log and routing could all use payload information with a single parse. | ||
| Also notably performing the parsing in payload passthrough buffer will mean deserialization once | ||
| and not re-serializing, which is the most performant outcome. | ||
|
|
||
| If any of the filter chain doesn't support payload passthrough, a customized non-passthrough | ||
| filter to setup metadata is encouraged from point of performance view. | ||
|
|
||
| Example | ||
| ------- | ||
|
|
||
| A sample filter configuration to route traffic to endpoints based on the presence or | ||
| absence of a version payload could be: | ||
|
|
||
| .. literalinclude:: _include/payload-to-metadata-filter.yaml | ||
| :language: yaml | ||
| :lines: 20-38 | ||
| :lineno-start: 20 | ||
| :linenos: | ||
| :caption: :download:`payload-to-metadata-filter.yaml <_include/payload-to-metadata-filter.yaml>` | ||
|
|
||
| A corresponding upstream cluster configuration could be: | ||
|
|
||
| .. literalinclude:: _include/header-to-metadata-filter.yaml | ||
| :language: yaml | ||
| :lines: 39-49 | ||
| :lineno-start: 37 | ||
| :linenos: | ||
| :caption: :download:`header-to-metadata-filter.yaml <_include/header-to-metadata-filter.yaml>` | ||
|
|
||
| The request thrift structure could be: | ||
|
|
||
| .. literalinclude:: _include/request.proto | ||
| :language: proto | ||
|
|
||
| This would then allow requests of method name ``foo`` with the ``version`` payload field which is | ||
| under ``info`` field set to be matched against endpoints with the corresponding version. Whereas | ||
| requests with that payload missing would be matched with the default endpoints. | ||
|
|
||
| The regex matching and substitution is similiar with :ref:`header to metadata filter <config_thrift_filters_header_to_metadata>`. | ||
|
|
||
|
|
||
| Statistics | ||
| ---------- | ||
|
|
||
| Currently, this filter generates no statistics. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
36 changes: 36 additions & 0 deletions
36
source/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/BUILD
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| load( | ||
| "//bazel:envoy_build_system.bzl", | ||
| "envoy_cc_extension", | ||
| "envoy_cc_library", | ||
| "envoy_extension_package", | ||
| ) | ||
|
|
||
| licenses(["notice"]) # Apache 2 | ||
|
|
||
| envoy_extension_package() | ||
|
|
||
| envoy_cc_extension( | ||
| name = "config", | ||
| srcs = ["config.cc"], | ||
| hdrs = ["config.h"], | ||
| deps = [ | ||
| ":payload_to_metadata_filter_lib", | ||
| "//envoy/registry", | ||
| "//source/extensions/filters/network/thrift_proxy/filters:factory_base_lib", | ||
| "@envoy_api//envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3:pkg_cc_proto", | ||
| ], | ||
| ) | ||
|
|
||
| envoy_cc_library( | ||
| name = "payload_to_metadata_filter_lib", | ||
| srcs = ["payload_to_metadata_filter.cc"], | ||
| hdrs = ["payload_to_metadata_filter.h"], | ||
| deps = [ | ||
| "//envoy/server:filter_config_interface", | ||
| "//source/extensions/filters/network/thrift_proxy:auto_protocol_lib", | ||
| "//source/extensions/filters/network/thrift_proxy:auto_transport_lib", | ||
| "//source/extensions/filters/network/thrift_proxy:decoder_lib", | ||
| "//source/extensions/filters/network/thrift_proxy/filters:pass_through_filter_lib", | ||
| "@envoy_api//envoy/extensions/filters/network/thrift_proxy/filters/payload_to_metadata/v3:pkg_cc_proto", | ||
| ], | ||
| ) |
Oops, something went wrong.