http: json_to_metadata filter

CODEOWNERS

@@ -284,6 +284,8 @@ extensions/filters/http/oauth2 @derekargueta @mattklein123
 /*/extensions/filters/http/ip_tagging @alyssawilk @JuniorHsu
 # Header to metadata
 /*/extensions/filters/http/header_to_metadata @zuercher @JuniorHsu
+# Json to metadata
+/*/extensions/filters/http/json_to_metadata @JuniorHsu @kbaichoo
 # zookeeper
 /*/extensions/filters/network/zookeeper_proxy @JuniorHsu @Winbobob @mattklein123
 # Custom response filter

api/BUILD

@@ -188,6 +188,7 @@ proto_library(
         "//envoy/extensions/filters/http/header_to_metadata/v3:pkg",
         "//envoy/extensions/filters/http/health_check/v3:pkg",
         "//envoy/extensions/filters/http/ip_tagging/v3:pkg",
+        "//envoy/extensions/filters/http/json_to_metadata/v3:pkg",
         "//envoy/extensions/filters/http/jwt_authn/v3:pkg",
         "//envoy/extensions/filters/http/kill_request/v3:pkg",
         "//envoy/extensions/filters/http/local_ratelimit/v3:pkg",

api/envoy/extensions/filters/http/json_to_metadata/v3/BUILD

@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+)

api/envoy/extensions/filters/http/json_to_metadata/v3/json_to_metadata.proto

@@ -0,0 +1,116 @@
+syntax = "proto3";
+
+package envoy.extensions.filters.http.json_to_metadata.v3;
+
+import "google/protobuf/struct.proto";
+
+import "udpa/annotations/status.proto";
+import "validate/validate.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.filters.http.json_to_metadata.v3";
+option java_outer_classname = "JsonToMetadataProto";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/json_to_metadata/v3;json_to_metadatav3";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Json-To-Metadata Filter]
+//
+// The configuration for transforming json body into metadata. This is useful
+// for matching load balancer subsets, logging, etc.
+//
+// Json to Metadata :ref:`configuration overview <config_http_filters_json_to_metadata>`.
+// [#extension: envoy.filters.http.json_to_metadata]
+
+message JsonToMetadata {
+  enum ValueType {
+    // The value is a serialized `protobuf.Value
+    // <https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/struct.proto#L62>`_.
+    PROTOBUF_VALUE = 0;
+
+    STRING = 1;
+
+    NUMBER = 2;
+  }
+
+  // [#next-free-field: 6]
+  message KeyValuePair {
+    // The namespace — if this is empty, the filter's namespace will be used.
+    string metadata_namespace = 1;
+
+    // The key to use within the namespace.
+    string key = 2 [(validate.rules).string = {min_len: 1}];
+
+    oneof value_type {
+      // The value to pair with the given key.
+      //
+      // When used for on_present case, if value is non-empty it'll be used instead
+      // of the the value of the JSON key. If both are empty, the the value of the
+      // JSON key is used as-is.
+      //
+      // When used for on_missing/on_error case, a non-empty value
+      // must be provided.
+      //
+      // It ignores ValueType, i.e., not type casting.
+      google.protobuf.Value value = 3;
+    }
+
+    // The value's type — defaults to protobuf.Value.
+    ValueType type = 4 [(validate.rules).enum = {defined_only: true}];
+
+    // False if we want to overwrite the existing metadata value. Default to false.
+    bool preserve_existing_metadata_value = 5;
+  }
+
+  message Selector {
+    // TODO(kuochunghsu): Explore matchers for array handling.
+    oneof selector {
+      // key to match
+      string key = 1 [(validate.rules).string = {min_len: 1}];
+    }
+  }
+
+  // A Rule defines what metadata to apply when a key-value is present, missing in the json
+  // or fail to parse the payload.
+  message Rule {
+    // Specifies that a match will be performed on the value of a property.
+    // Here's an example to match on 1 in {"foo": {"bar": 1}, "bar": 2}
+    //
+    // selectors:
+    // - key: foo
+    // - key: bar
+    repeated Selector selectors = 1 [(validate.rules).repeated = {min_items: 1}];
+
+    // If the attribute is present, apply this metadata KeyValuePair.
+    KeyValuePair on_present = 2;
+
+    // If the attribute is missing, apply this metadata KeyValuePair.
+    //
+    // The value in the KeyValuePair must be set.
+    KeyValuePair on_missing = 3;
+
+    // If the body is too large or fail to parse, apply this metadata KeyValuePair.
+    //
+    // The value in the KeyValuePair must be set.
+    KeyValuePair on_error = 4;
+  }
+
+  message MatchRules {
+    // The list of rules to apply.
+    repeated Rule rules = 1 [(validate.rules).repeated = {min_items: 1}];
+
+    // Allowed content-type for json to metadata transformation.
+    // Default to {"application/json"}.
+    //
+    // Set `allow_empty_content_type` if empty/missing content-type header
+    // is allowed.
+    repeated string allow_content_types = 2
+        [(validate.rules).repeated = {items {string {min_len: 1}}}];
+
+    // Allowed empty content-type for json to metadata transformation.
+    // Default to false.
+    bool allow_empty_content_type = 3;
+  }
+
+  // Rules to match json body of requests
+  MatchRules request_rules = 1 [(validate.rules).message = {required: true}];
+}

api/versioning/BUILD

@@ -126,6 +126,7 @@ proto_library(
         "//envoy/extensions/filters/http/header_to_metadata/v3:pkg",
         "//envoy/extensions/filters/http/health_check/v3:pkg",
         "//envoy/extensions/filters/http/ip_tagging/v3:pkg",
+        "//envoy/extensions/filters/http/json_to_metadata/v3:pkg",
         "//envoy/extensions/filters/http/jwt_authn/v3:pkg",
         "//envoy/extensions/filters/http/kill_request/v3:pkg",
         "//envoy/extensions/filters/http/local_ratelimit/v3:pkg",

changelogs/current.yaml

@@ -305,6 +305,9 @@ new_features:
   change: |
     added support for CONNECT-UDP (RFC 9298). Can be disabled by setting runtime feature
     ``envoy.reloadable_features.enable_connect_udp_support`` to false.
+- area: http
+  change: |
+    added :ref:`Json-To-Metadata filter <envoy_v3_api_msg_extensions.filters.http.json_to_metadata.v3.JsonToMetadata>`.
 - area: listeners
   change: |
     added :ref:`max_connections_to_accept_per_socket_event

docs/root/configuration/http/http_filters/_include/json-to-metadata-filter.yaml

@@ -0,0 +1,83 @@
+static_resources:
+  listeners:
+  - address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 80
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          codec_type: AUTO
+          stat_prefix: ingress_http
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: app
+              domains:
+              - "*"
+              routes:
+              - match:
+                  prefix: "/"
+                route:
+                  cluster: versioned-cluster
+          http_filters:
+          - name: envoy.filters.http.json_to_metadata
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.json_to_metadata.v3.JsonToMetadata
+              request_rules:
+                rules:
+                - selectors:
+                  - key: version
+                  on_present:
+                    metadata_namespace: envoy.lb
+                    key: version
+                  on_missing:
+                    metadata_namespace: envoy.lb
+                    key: default
+                    value: "true"
+                    preserve_existing_metadata_value: true
+                  on_error:
+                    metadata_namespace: envoy.lb
+                    key: default
+                    value: "true"
+                    preserve_existing_metadata_value: true
+          - name: envoy.filters.http.router
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+
+  clusters:
+  - name: versioned-cluster
+    type: STRICT_DNS
+    lb_policy: ROUND_ROBIN
+    lb_subset_config:
+      fallback_policy: ANY_ENDPOINT
+      subset_selectors:
+      - keys:
+        - default
+      - keys:
+        - version
+    load_assignment:
+      cluster_name: versioned-cluster
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: 127.0.0.1
+                port_value: 8080
+          metadata:
+            filter_metadata:
+              envoy.lb:
+                default: "true"
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: 127.0.0.1
+                port_value: 8081
+          metadata:
+            filter_metadata:
+              envoy.lb:
+                version: "1.0"

docs/root/configuration/http/http_filters/http_filters.rst

@@ -40,6 +40,7 @@ HTTP filters
   health_check_filter
   header_to_metadata_filter
   ip_tagging_filter
+  json_to_metadata_filter
   jwt_authn_filter
   kill_request_filter
   language_filter

docs/root/configuration/http/http_filters/json_to_metadata_filter.rst

@@ -0,0 +1,52 @@
+.. _config_http_filters_json_to_metadata:
+
+Envoy Json-To-Metadata Filter
+=============================
+* This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.json_to_metadata.v3.JsonToMetadata``.
+* :ref:`v3 API reference <envoy_v3_api_msg_extensions.filters.http.json_to_metadata.v3.JsonToMetadata>`
+
+This filter is configured with rules that will be matched against requests.
+Each rule has either a series of selectors and can be triggered either when the json attribute
+is present or missing.
+
+When a rule is triggered, dynamic metadata will be added based on the configuration of the rule.
+If present, it's value is extracted and used along with the specified key as metadata. If missing,
+on missing case is triggered and the value specified is used for adding metadata.
+
+The metadata can then be used for load balancing decisions, consumed from logs, etc.
+
+A typical use case for this filter is to dynamically match a specified json attribute of requests
+with rate limit. For this, a given value of the JSON keys would be extracted and attached
+to the request as dynamic metadata which would then be used to match a rate limit action on metadata.
+
+The Json to metadata filter stops iterating to next filter until we have the whole payload to extract
+the Json attributes or encounter error.
+
+Example
+-------
+
+A sample filter configuration to route traffic to endpoints based on the presence or
+absence of a version attribute could be:
+
+.. literalinclude:: _include/json-to-metadata-filter.yaml
+    :language: yaml
+    :lines: 25-45
+    :lineno-start: 25
+    :linenos:
+    :caption: :download:`json-to-metadata-filter.yaml <_include/json-to-metadata-filter.yaml>`
+
+Statistics
+----------
+
+The json to metadata filter outputs statistics in the *http.<stat_prefix>.json_to_metadata.* namespace. The :ref:`stat prefix
+<envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.stat_prefix>`
+comes from the owning HTTP connection manager.
+
+.. csv-table::
+  :header: Name, Type, Description
+  :widths: 1, 1, 2
+
+  rq_success, Counter, Total requests that succeed to parse the json body
+  rq_mismatched_content_type, Counter, Total requests that mismatch the content type
+  rq_no_body, Counter, Total requests without content body
+  rq_invalid_json_body, Counter, Total requests with invalid json body

source/extensions/extensions_build_config.bzl

@@ -150,6 +150,7 @@ EXTENSIONS = {
     "envoy.filters.http.header_to_metadata":            "//source/extensions/filters/http/header_to_metadata:config",
     "envoy.filters.http.health_check":                  "//source/extensions/filters/http/health_check:config",
     "envoy.filters.http.ip_tagging":                    "//source/extensions/filters/http/ip_tagging:config",
+    "envoy.filters.http.json_to_metadata":            "//source/extensions/filters/http/json_to_metadata:config",
     "envoy.filters.http.jwt_authn":                     "//source/extensions/filters/http/jwt_authn:config",
     "envoy.filters.http.rate_limit_quota":              "//source/extensions/filters/http/rate_limit_quota:config",
     # Disabled by default

source/extensions/extensions_metadata.yaml

@@ -406,6 +406,13 @@ envoy.filters.http.ip_tagging:
   status: stable
   type_urls:
   - envoy.extensions.filters.http.ip_tagging.v3.IPTagging
+envoy.filters.http.json_to_metadata:
+  categories:
+  - envoy.filters.http
+  security_posture: robust_to_untrusted_downstream
+  status: alpha
+  type_urls:
+  - envoy.extensions.filters.http.json_to_metadata.v3.JsonToMetadata
 envoy.filters.http.jwt_authn:
   categories:
   - envoy.filters.http

source/extensions/filters/http/json_to_metadata/BUILD

@@ -0,0 +1,35 @@
+load(
+    "//bazel:envoy_build_system.bzl",
+    "envoy_cc_extension",
+    "envoy_cc_library",
+    "envoy_extension_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_extension_package()
+
+envoy_cc_library(
+    name = "json_to_metadata_lib",
+    srcs = ["filter.cc"],
+    hdrs = ["filter.h"],
+    deps = [
+        "//envoy/server:filter_config_interface",
+        "//source/common/http:header_utility_lib",
+        "//source/common/json:json_loader_lib",
+        "//source/extensions/filters/http/common:pass_through_filter_lib",
+        "@envoy_api//envoy/extensions/filters/http/json_to_metadata/v3:pkg_cc_proto",
+    ],
+)
+
+envoy_cc_extension(
+    name = "config",
+    srcs = ["config.cc"],
+    hdrs = ["config.h"],
+    deps = [
+        ":json_to_metadata_lib",
+        "//envoy/registry",
+        "//source/extensions/filters/http/common:factory_base_lib",
+        "@envoy_api//envoy/extensions/filters/http/json_to_metadata/v3:pkg_cc_proto",
+    ],
+)