config: add support Any as opaque config (envoyproxy#5435)

Add support of Any as opaque config for extensions. Deprecates Struct configs. Fixes envoyproxy#4475.

Risk Level: Low
Testing: CI
Docs Changes: Added.
Release Notes: Added.

Signed-off-by: Lizan Zhou <[email protected]>
Signed-off-by: Fred Douglas <[email protected]>

DEPRECATED.md

@@ -10,8 +10,10 @@ A logged warning is expected for each deprecated item that is in deprecation win
 * Use of `enabled` in `CorsPolicy`, found in
   [route.proto](https://github.com/envoyproxy/envoy/blob/master/api/envoy/api/v2/route/route.proto).
   Set the `filter_enabled` field instead.
+* Use of google.protobuf.Struct for extension opaque configs is deprecated. Use google.protobuf.Any instead or pack
+google.protobuf.Struct in google.protobuf.Any.
 
-## Version 1.9.0
+## Version 1.9.0 (Dec 20, 2018)
 
 * Order of execution of the network write filter chain has been reversed. Prior to this release cycle it was incorrect, see [#4599](https://github.com/envoyproxy/envoy/issues/4599). In the 1.9.0 release cycle we introduced `bugfix_reverse_write_filter_order` in [lds.proto] (https://github.com/envoyproxy/envoy/blob/master/api/envoy/api/v2/lds.proto) to temporarily support both old and new behaviors. Note this boolean field is deprecated.
 * Order of execution of the HTTP encoder filter chain has been reversed. Prior to this release cycle it was incorrect, see [#4599](https://github.com/envoyproxy/envoy/issues/4599). In the 1.9.0 release cycle we introduced `bugfix_reverse_encode_order` in [http_connection_manager.proto] (https://github.com/envoyproxy/envoy/blob/master/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto) to temporarily support both old and new behaviors. Note this boolean field is deprecated.

api/envoy/api/v2/cds.proto

@@ -233,9 +233,8 @@ message Cluster {
   // for upstream connections. The key should match the extension filter name, such as
   // "envoy.filters.network.thrift_proxy". See the extension's documentation for details on
   // specific options.
-  map<string, google.protobuf.Struct> extension_protocol_options = 35;
+  map<string, google.protobuf.Struct> extension_protocol_options = 35 [deprecated = true];
 
-  // [#not-implemented-hide:]
   // The extension_protocol_options field is used to provide extension-specific protocol options
   // for upstream connections. The key should match the extension filter name, such as
   // "envoy.filters.network.thrift_proxy". See the extension's documentation for details on

api/envoy/api/v2/core/base.proto

@@ -190,9 +190,8 @@ message TransportSocket {
   // Implementation specific configuration which depends on the implementation being instantiated.
   // See the supported transport socket implementations for further documentation.
   oneof config_type {
-    google.protobuf.Struct config = 2;
+    google.protobuf.Struct config = 2 [deprecated = true];
 
-    // [#not-implemented-hide:]
     google.protobuf.Any typed_config = 3;
   }
 }

api/envoy/api/v2/core/grpc_service.proto

@@ -82,9 +82,8 @@ message GrpcService {
       message MetadataCredentialsFromPlugin {
         string name = 1;
         oneof config_type {
-          google.protobuf.Struct config = 2;
+          google.protobuf.Struct config = 2 [deprecated = true];
 
-          // [#not-implemented-hide:]
           google.protobuf.Any typed_config = 3;
         }
       }

api/envoy/api/v2/core/health_check.proto

@@ -163,9 +163,8 @@ message HealthCheck {
     // A custom health checker specific configuration which depends on the custom health checker
     // being instantiated. See :api:`envoy/config/health_checker` for reference.
     oneof config_type {
-      google.protobuf.Struct config = 2;
+      google.protobuf.Struct config = 2 [deprecated = true];
 
-      // [#not-implemented-hide:]
       google.protobuf.Any typed_config = 3;
     }
   }

api/envoy/api/v2/listener/listener.proto

@@ -39,9 +39,8 @@ message Filter {
   // Filter specific configuration which depends on the filter being
   // instantiated. See the supported filters for further documentation.
   oneof config_type {
-    google.protobuf.Struct config = 2;
+    google.protobuf.Struct config = 2 [deprecated = true];
 
-    // [#not-implemented-hide:]
     google.protobuf.Any typed_config = 4;
   }
 
@@ -209,9 +208,8 @@ message ListenerFilter {
   // Filter specific configuration which depends on the filter being instantiated.
   // See the supported filters for further documentation.
   oneof config_type {
-    google.protobuf.Struct config = 2;
+    google.protobuf.Struct config = 2 [deprecated = true];
 
-    // [#not-implemented-hide:]
     google.protobuf.Any typed_config = 3;
   }
 }

api/envoy/api/v2/route/route.proto

@@ -113,9 +113,8 @@ message VirtualHost {
   // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter
   // specific; see the :ref:`HTTP filter documentation <config_http_filters>`
   // for if and how it is utilized.
-  map<string, google.protobuf.Struct> per_filter_config = 12;
+  map<string, google.protobuf.Struct> per_filter_config = 12 [deprecated = true];
 
-  // [#not-implemented-hide:]
   // The per_filter_config field can be used to provide virtual host-specific
   // configurations for filters. The key should match the filter name, such as
   // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter
@@ -181,9 +180,8 @@ message Route {
   // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter
   // specific; see the :ref:`HTTP filter documentation <config_http_filters>` for
   // if and how it is utilized.
-  map<string, google.protobuf.Struct> per_filter_config = 8;
+  map<string, google.protobuf.Struct> per_filter_config = 8 [deprecated = true];
 
-  // [#not-implemented-hide:]
   // The per_filter_config field can be used to provide route-specific
   // configurations for filters. The key should match the filter name, such as
   // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter
@@ -279,9 +277,8 @@ message WeightedCluster {
     // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter
     // specific; see the :ref:`HTTP filter documentation <config_http_filters>`
     // for if and how it is utilized.
-    map<string, google.protobuf.Struct> per_filter_config = 8;
+    map<string, google.protobuf.Struct> per_filter_config = 8 [deprecated = true];
 
-    // [#not-implemented-hide:]
     // The per_filter_config field can be used to provide weighted cluster-specific
     // configurations for filters. The key should match the filter name, such as
     // *envoy.buffer* for the HTTP buffer filter. Use of this field is filter
@@ -805,9 +802,8 @@ message RetryPolicy {
   message RetryPriority {
     string name = 1 [(validate.rules).string.min_bytes = 1];
     oneof config_type {
-      google.protobuf.Struct config = 2;
+      google.protobuf.Struct config = 2 [deprecated = true];
 
-      // [#not-implemented-hide:]
       google.protobuf.Any typed_config = 3;
     }
   }
@@ -820,9 +816,8 @@ message RetryPolicy {
   message RetryHostPredicate {
     string name = 1 [(validate.rules).string.min_bytes = 1];
     oneof config_type {
-      google.protobuf.Struct config = 2;
+      google.protobuf.Struct config = 2 [deprecated = true];
 
-      // [#not-implemented-hide:]
       google.protobuf.Any typed_config = 3;
     }
   }

api/envoy/config/filter/accesslog/v2/accesslog.proto

@@ -35,9 +35,8 @@ message AccessLog {
   // #. "envoy.http_grpc_access_log": :ref:`HttpGrpcAccessLogConfig
   //    <envoy_api_msg_config.accesslog.v2.HttpGrpcAccessLogConfig>`
   oneof config_type {
-    google.protobuf.Struct config = 3;
+    google.protobuf.Struct config = 3 [deprecated = true];
 
-    // [#not-implemented-hide:]
     google.protobuf.Any typed_config = 4;
   }
 }

api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto

@@ -410,9 +410,8 @@ message HttpFilter {
   // Filter specific configuration which depends on the filter being instantiated. See the supported
   // filters for further documentation.
   oneof config_type {
-    google.protobuf.Struct config = 2;
+    google.protobuf.Struct config = 2 [deprecated = true];
 
-    // [#not-implemented-hide:]
     google.protobuf.Any typed_config = 4;
   }
 

api/envoy/config/filter/network/thrift_proxy/v2alpha1/thrift_proxy.proto

@@ -95,9 +95,8 @@ message ThriftFilter {
   // Filter specific configuration which depends on the filter being instantiated. See the supported
   // filters for further documentation.
   oneof config_type {
-    google.protobuf.Struct config = 2;
+    google.protobuf.Struct config = 2 [deprecated = true];
 
-    // [#not-implemented-hide:]
     google.protobuf.Any typed_config = 3;
   }
 }

api/envoy/config/metrics/v2/stats.proto

@@ -33,9 +33,8 @@ message StatsSink {
   // Stats sink specific configuration which depends on the sink being instantiated. See
   // :ref:`StatsdSink <envoy_api_msg_config.metrics.v2.StatsdSink>` for an example.
   oneof config_type {
-    google.protobuf.Struct config = 2;
+    google.protobuf.Struct config = 2 [deprecated = true];
 
-    // [#not-implemented-hide:]
     google.protobuf.Any typed_config = 3;
   }
 }

api/envoy/config/overload/v2alpha/overload.proto

@@ -30,9 +30,8 @@ message ResourceMonitor {
 
   // Configuration for the resource monitor being instantiated.
   oneof config_type {
-    google.protobuf.Struct config = 2;
+    google.protobuf.Struct config = 2 [deprecated = true];
 
-    // [#not-implemented-hide:]
     google.protobuf.Any typed_config = 3;
   }
 }

api/envoy/config/trace/v2/trace.proto

@@ -41,9 +41,8 @@ message Tracing {
     // - :ref:`DynamicOtConfig <envoy_api_msg_config.trace.v2.DynamicOtConfig>`
     // - :ref:`DatadogConfig <envoy_api_msg_config.trace.v2.DatadogConfig>`
     oneof config_type {
-      google.protobuf.Struct config = 2;
+      google.protobuf.Struct config = 2 [deprecated = true];
 
-      // [#not-implemented-hide:]
       google.protobuf.Any typed_config = 3;
     }
   }

docs/root/intro/version_history.rst

@@ -3,6 +3,7 @@ Version history
 
 1.10.0 (pending)
 ================
+* config: added support of using google.protobuf.Any in opaque configs for extensions.
 * access log: added a new flag for upstream retry count exceeded.
 * admin: the admin server can now be accessed via HTTP/2 (prior knowledge).
 * buffer: fix vulnerabilities when allocation fails

source/common/config/utility.cc

@@ -278,5 +278,36 @@ envoy::api::v2::ClusterLoadAssignment Utility::translateClusterHosts(
   return load_assignment;
 }
 
+void Utility::translateOpaqueConfig(const ProtobufWkt::Any& typed_config,
+                                    const ProtobufWkt::Struct& config,
+                                    Protobuf::Message& out_proto) {
+  static const std::string& struct_type =
+      ProtobufWkt::Struct::default_instance().GetDescriptor()->full_name();
+
+  if (!typed_config.value().empty()) {
+
+    // Unpack methods will only use the fully qualified type name after the last '/'.
+    // https://github.com/protocolbuffers/protobuf/blob/3.6.x/src/google/protobuf/any.proto#L87
+    absl::string_view type = typed_config.type_url();
+    size_t pos = type.find_last_of('/');
+    if (pos != absl::string_view::npos) {
+      type = type.substr(pos + 1);
+    }
+
+    // out_proto is expecting Struct, unpack directly
+    if (type != struct_type || out_proto.GetDescriptor()->full_name() == struct_type) {
+      typed_config.UnpackTo(&out_proto);
+    } else {
+      ProtobufWkt::Struct struct_config;
+      typed_config.UnpackTo(&struct_config);
+      MessageUtil::jsonConvert(struct_config, out_proto);
+    }
+  }
+
+  if (!config.fields().empty()) {
+    MessageUtil::jsonConvert(config, out_proto);
+  }
+}
+
 } // namespace Config
 } // namespace Envoy

source/common/config/utility.h

@@ -280,55 +280,11 @@ class Utility {
     // Fail in an obvious way if a plugin does not return a proto.
     RELEASE_ASSERT(config != nullptr, "");
 
-    if (enclosing_message.has_config()) {
-      MessageUtil::jsonConvert(enclosing_message.config(), *config);
-    }
+    translateOpaqueConfig(enclosing_message.typed_config(), enclosing_message.config(), *config);
 
     return config;
   }
 
-  /**
-   * Translate a nested config into a route-specific proto message provided by
-   * the implementation factory.
-   * @param source Protobuf::Message containing the opaque config for the given factory's
-   *        route-local configuration.
-   * @param factory Server::Configuration::NamedHttpFilterConfigFactory implementation
-   * @return ProtobufTypes::MessagePtr the translated config
-   */
-  static ProtobufTypes::MessagePtr
-  translateToFactoryRouteConfig(const Protobuf::Message& source,
-                                Server::Configuration::NamedHttpFilterConfigFactory& factory) {
-    ProtobufTypes::MessagePtr config = factory.createEmptyRouteConfigProto();
-
-    // Fail in an obvious way if a plugin does not return a proto.
-    RELEASE_ASSERT(config != nullptr, "");
-
-    MessageUtil::jsonConvert(source, *config);
-    return config;
-  }
-
-  /**
-   * Translate a nested config into a protocol-specific options proto message provided by the
-   * implementation factory.
-   * @param source Protobuf::Message containing the opaque config for the given factory's
-   *        protocol specific configuration.
-   * @param factory Server::Configuration::NamedNetworkFilterConfigFactory implementation
-   * @return ProtobufTypes::MessagePtr the translated config
-   * @throws EnvoyException if the factory does not support protocol options
-   */
-  static ProtobufTypes::MessagePtr
-  translateToFactoryProtocolOptionsConfig(const Protobuf::Message& source, const std::string& name,
-                                          Server::Configuration::ProtocolOptionsFactory& factory) {
-    ProtobufTypes::MessagePtr config = factory.createEmptyProtocolOptionsProto();
-
-    if (config == nullptr) {
-      throw EnvoyException(fmt::format("filter {} does not support protocol options", name));
-    }
-
-    MessageUtil::jsonConvert(source, *config);
-    return config;
-  }
-
   /**
    * Create TagProducer instance. Check all tag names for conflicts to avoid
    * unexpected tag name overwriting.
@@ -373,6 +329,17 @@ class Utility {
    */
   static envoy::api::v2::ClusterLoadAssignment
   translateClusterHosts(const Protobuf::RepeatedPtrField<envoy::api::v2::core::Address>& hosts);
+
+  /**
+   * Translate opaque config from google.protobuf.Any or google.protobuf.Struct to defined proto
+   * message.
+   * @param typed_config opaque config packed in google.protobuf.Any
+   * @param config the deprecated google.protobuf.Struct config, empty struct if doesn't exist.
+   * @param out_proto the proto message instantiated by extensions
+   */
+  static void translateOpaqueConfig(const ProtobufWkt::Any& typed_config,
+                                    const ProtobufWkt::Struct& config,
+                                    Protobuf::Message& out_proto);
 };
 
 } // namespace Config