hc: add health event sink interface and file sink implementation

CODEOWNERS

@@ -308,6 +308,8 @@ extensions/filters/http/oauth2 @derekargueta @snowp
 /*/extensions/health_checkers/grpc @snowp @zuercher
 /*/extensions/health_checkers/http @snowp @zuercher
 /*/extensions/health_checkers/tcp @snowp @zuercher
+# Health check event sinks
+/*/extensions/health_check_event_sinks/file @botengyao @yanavlasov
 # IP Geolocation
 /*/extensions/filters/http/geoip @nezdolik @ravenblackx
 

api/BUILD

@@ -235,6 +235,7 @@ proto_library(
         "//envoy/extensions/formatter/cel/v3:pkg",
         "//envoy/extensions/formatter/metadata/v3:pkg",
         "//envoy/extensions/formatter/req_without_query/v3:pkg",
+        "//envoy/extensions/health_check_event_sinks/file/v3:pkg",
         "//envoy/extensions/health_checkers/redis/v3:pkg",
         "//envoy/extensions/health_checkers/thrift/v3:pkg",
         "//envoy/extensions/http/cache/file_system_http_cache/v3:pkg",

api/envoy/config/core/v3/health_check.proto

@@ -4,6 +4,7 @@ package envoy.config.core.v3;
 
 import "envoy/config/core/v3/base.proto";
 import "envoy/config/core/v3/event_service_config.proto";
+import "envoy/config/core/v3/extension.proto";
 import "envoy/type/matcher/v3/string.proto";
 import "envoy/type/v3/http.proto";
 import "envoy/type/v3/range.proto";
@@ -13,6 +14,7 @@ import "google/protobuf/duration.proto";
 import "google/protobuf/struct.proto";
 import "google/protobuf/wrappers.proto";
 
+import "envoy/annotations/deprecation.proto";
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
 import "validate/validate.proto";
@@ -60,7 +62,7 @@ message HealthStatusSet {
       [(validate.rules).repeated = {items {enum {defined_only: true}}}];
 }
 
-// [#next-free-field: 25]
+// [#next-free-field: 26]
 message HealthCheck {
   option (udpa.annotations.versioning).previous_message_type = "envoy.api.v2.core.HealthCheck";
 
@@ -366,9 +368,19 @@ message HealthCheck {
   // The default value for "healthy edge interval" is the same as the default interval.
   google.protobuf.Duration healthy_edge_interval = 16 [(validate.rules).duration = {gt {}}];
 
+  // .. attention::
+  // This field is deprecated in favor of the extension
+  // :ref:`event_logger <envoy_v3_api_field_config.core.v3.HealthCheck.event_logger>` and
+  // :ref:`event_log_path <envoy_v3_api_field_extensions.health_check_event_sinks.file.v3.HealthCheckEventFileSink.event_log_path>`
+  // in the file sink extension.
+  //
   // Specifies the path to the :ref:`health check event log <arch_overview_health_check_logging>`.
-  // If empty, no event log will be written.
-  string event_log_path = 17;
+  string event_log_path = 17
+      [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"];
+
+  // A list of event log sinks to process the health check event.
+  // [#extension-category: envoy.health_check_event_sinks]
+  repeated TypedExtensionConfig event_logger = 25;
 
   // [#not-implemented-hide:]
   // The gRPC service for the health check event service.

api/envoy/extensions/health_check_event_sinks/file/v3/BUILD

@@ -0,0 +1,9 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = ["@com_github_cncf_udpa//udpa/annotations:pkg"],
+)

api/envoy/extensions/health_check_event_sinks/file/v3/file.proto

@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+package envoy.extensions.health_check_event_sinks.file.v3;
+
+import "udpa/annotations/status.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.health_check_event_sinks.file.v3";
+option java_outer_classname = "FileProto";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/health_check_event_sinks/file/v3;filev3";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: Health Check Log File Sink]
+// [#extension: envoy.health_check_event_sinks.file]
+
+// Health check event file sink.
+// The health check event will be converted to JSON.
+message HealthCheckEventFileSink {
+  // Specifies the path to the health check event log.
+  string event_log_path = 1;
+}

api/versioning/BUILD

@@ -173,6 +173,7 @@ proto_library(
         "//envoy/extensions/formatter/cel/v3:pkg",
         "//envoy/extensions/formatter/metadata/v3:pkg",
         "//envoy/extensions/formatter/req_without_query/v3:pkg",
+        "//envoy/extensions/health_check_event_sinks/file/v3:pkg",
         "//envoy/extensions/health_checkers/redis/v3:pkg",
         "//envoy/extensions/health_checkers/thrift/v3:pkg",
         "//envoy/extensions/http/cache/file_system_http_cache/v3:pkg",

docs/root/api-v3/config/health_check_event_sinks/health_check_event_sinks.rst

@@ -0,0 +1,10 @@
+.. _api_v3_health_check_event_sinks:
+
+Health check event sinks
+========================
+
+.. toctree::
+  :glob:
+  :maxdepth: 2
+
+  ../../extensions/health_check_event_sinks/*/v3/*

docs/root/intro/arch_overview/upstream/health_checking.rst

@@ -85,13 +85,22 @@ Health check event logging
 --------------------------
 
 A per-healthchecker log of ejection and addition events can optionally be produced by Envoy by
-specifying a log file path in :ref:`the HealthCheck config <envoy_v3_api_field_config.core.v3.HealthCheck.event_log_path>`.
+specifying a log file path in :ref:`the HealthCheck config event_log_path <envoy_v3_api_field_config.core.v3.HealthCheck.event_log_path>`.
 The log is structured as JSON dumps of
 :ref:`HealthCheckEvent messages <envoy_v3_api_msg_data.core.v3.HealthCheckEvent>`.
 
+Note: :ref:`the HealthCheck config event_log_path <envoy_v3_api_field_config.core.v3.HealthCheck.event_log_path>` is deperated in favor of 
+:ref:`HealthCheck event_logger extension <envoy_v3_api_field_config.core.v3.HealthCheck.event_logger>`.
+The :ref:`event_log_path <envoy_v3_api_field_extensions.health_check_event_sinks.file.v3.HealthCheckEventFileSink.event_log_path>` is used in the file sink extension for the JSON dumps.
+
+A new event sink extension catalog
+`envoy.health_check_event_sinks` is created, and APIs can be found :ref:`here <api_v3_health_check_event_sinks>`.
+
 Envoy can be configured to log all health check failure events by setting the :ref:`always_log_health_check_failures
 flag <envoy_v3_api_field_config.core.v3.HealthCheck.always_log_health_check_failures>` to true.
 
+
+
 Passive health checking
 -----------------------
 

envoy/server/health_checker_config.h

@@ -46,6 +46,13 @@ class HealthCheckerFactoryContext {
    * @return Api::Api& the API used by the server.
    */
   virtual Api::Api& api() PURE;
+
+  /**
+   * @return AccessLogManager for use by the entire server.
+   */
+  virtual AccessLog::AccessLogManager& accessLogManager() PURE;
+
+  virtual void setEventLogger(Upstream::HealthCheckEventLoggerPtr event_logger) PURE;
 };
 
 /**

envoy/upstream/BUILD

@@ -41,6 +41,15 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "health_check_event_sink_interface",
+    hdrs = ["health_check_event_sink.h"],
+    deps = [
+        "//envoy/config:typed_config_interface",
+        "//envoy/server:health_checker_config_interface",
+    ],
+)
+
 envoy_cc_library(
     name = "health_checker_interface",
     hdrs = ["health_checker.h"],

envoy/upstream/health_check_event_sink.h

@@ -0,0 +1,38 @@
+#include "envoy/config/typed_config.h"
+#include "envoy/server/health_checker_config.h"
+
+namespace Envoy {
+namespace Upstream {
+
+/**
+ * Sink for health check event.
+ */
+class HealthCheckEventSink {
+public:
+  virtual ~HealthCheckEventSink() = default;
+
+  virtual void log(envoy::data::core::v3::HealthCheckEvent event) PURE;
+};
+
+using HealthCheckEventSinkPtr = std::unique_ptr<HealthCheckEventSink>;
+using HealthCheckEventSinkSharedPtr = std::shared_ptr<HealthCheckEventSink>;
+
+/**
+ * A factory abstract class for creating instances of HealthCheckEventSink.
+ */
+class HealthCheckEventSinkFactory : public Config::TypedFactory {
+public:
+  ~HealthCheckEventSinkFactory() override = default;
+
+  /**
+   * Creates an HealthCheckEventSink using the given config.
+   */
+  virtual HealthCheckEventSinkPtr
+  createHealthCheckEventSink(const ProtobufWkt::Any& config,
+                             Server::Configuration::HealthCheckerFactoryContext& context) PURE;
+
+  std::string category() const override { return "envoy.health_check_event_sink"; }
+};
+
+} // namespace Upstream
+} // namespace Envoy

source/common/upstream/BUILD

@@ -209,8 +209,14 @@ envoy_cc_library(
     srcs = ["health_checker_event_logger.cc"],
     hdrs = ["health_checker_event_logger.h"],
     deps = [
+        "//envoy/server:factory_context_interface",
+        "//envoy/server:health_checker_config_interface",
+        "//envoy/upstream:health_check_event_sink_interface",
         "//envoy/upstream:health_checker_interface",
+        "//source/common/access_log:access_log_lib",
         "//source/common/network:utility_lib",
+        "//source/common/protobuf:utility_lib",
+        "@envoy_api//envoy/config/accesslog/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
         "@envoy_api//envoy/data/core/v3:pkg_cc_proto",
         "@envoy_api//envoy/type/matcher:pkg_cc_proto",
@@ -231,6 +237,7 @@ envoy_cc_library(
         "@envoy_api//envoy/type/v3:pkg_cc_proto",
         # TODO(dio): Remove dependency to server.
         "//envoy/server:health_checker_config_interface",
+        "//envoy/server:factory_context_interface",
         "//source/common/grpc:codec_lib",
         "//source/common/router:router_lib",
         "//source/common/http:codec_client_lib",

source/common/upstream/health_checker_event_logger.cc

@@ -65,13 +65,19 @@ void HealthCheckEventLoggerImpl::createHealthCheckEvent(
   TimestampUtil::systemClockToTimestamp(time_source_.systemTime(), *event.mutable_timestamp());
 
   callback(event);
-#ifdef ENVOY_ENABLE_YAML
+  for (const auto& event_sink : event_sinks_) {
+    event_sink->log(event);
+  }
 
+#ifdef ENVOY_ENABLE_YAML
+  if (file_ == nullptr)
+    return;
   // Make sure the type enums make it into the JSON
   const auto json =
       MessageUtil::getJsonStringFromMessageOrError(event, /* pretty_print */ false,
                                                    /* always_print_primitive_fields */ true);
   file_->write(fmt::format("{}\n", json));
+
 #endif
 }
 } // namespace Upstream

source/common/upstream/health_checker_event_logger.h

@@ -2,25 +2,47 @@
 
 #include "envoy/access_log/access_log.h"
 #include "envoy/common/callback.h"
+#include "envoy/config/accesslog/v3/accesslog.pb.h"
+#include "envoy/config/accesslog/v3/accesslog.pb.validate.h"
 #include "envoy/config/core/v3/health_check.pb.h"
 #include "envoy/data/core/v3/health_check_event.pb.h"
 #include "envoy/event/timer.h"
 #include "envoy/runtime/runtime.h"
+#include "envoy/server/factory_context.h"
+#include "envoy/server/health_checker_config.h"
 #include "envoy/stats/scope.h"
 #include "envoy/type/matcher/string.pb.h"
+#include "envoy/upstream/health_check_event_sink.h"
 #include "envoy/upstream/health_checker.h"
 
+#include "source/common/access_log/access_log_impl.h"
 #include "source/common/common/logger.h"
+#include "source/common/config/utility.h"
+#include "source/common/protobuf/utility.h"
 
 namespace Envoy {
 namespace Upstream {
 
 class HealthCheckEventLoggerImpl : public HealthCheckEventLogger {
 public:
   HealthCheckEventLoggerImpl(AccessLog::AccessLogManager& log_manager, TimeSource& time_source,
-                             const std::string& file_name)
-      : time_source_(time_source), file_(log_manager.createAccessLog(Filesystem::FilePathAndType{
-                                       Filesystem::DestinationType::File, file_name})) {}
+                             const envoy::config::core::v3::HealthCheck& health_check_config,
+                             Server::Configuration::HealthCheckerFactoryContext& context)
+      : time_source_(time_source) {
+    if (!health_check_config.event_log_path().empty() /* deprecated */) {
+      file_ = log_manager.createAccessLog(Filesystem::FilePathAndType{
+          Filesystem::DestinationType::File, health_check_config.event_log_path()});
+    }
+
+    for (const auto& config : health_check_config.event_logger()) {
+      if (!config.has_typed_config())
+        continue;
+      auto& event_sink_factory =
+          Config::Utility::getAndCheckFactory<HealthCheckEventSinkFactory>(config);
+      event_sinks_.emplace_back(
+          event_sink_factory.createHealthCheckEventSink(config.typed_config(), context));
+    }
+  }
 
   void logEjectUnhealthy(envoy::data::core::v3::HealthCheckerType health_checker_type,
                          const HostDescriptionConstSharedPtr& host,
@@ -36,12 +58,16 @@ class HealthCheckEventLoggerImpl : public HealthCheckEventLogger {
   void logNoLongerDegraded(envoy::data::core::v3::HealthCheckerType health_checker_type,
                            const HostDescriptionConstSharedPtr& host) override;
 
+  std::vector<HealthCheckEventSinkSharedPtr> eventSinks() const { return event_sinks_; };
+
 private:
   void createHealthCheckEvent(
       envoy::data::core::v3::HealthCheckerType health_checker_type, const HostDescription& host,
       std::function<void(envoy::data::core::v3::HealthCheckEvent&)> callback) const;
   TimeSource& time_source_;
+  // Server::Configuration::HealthCheckerFactoryContext& context_;
   AccessLog::AccessLogFileSharedPtr file_;
+  std::vector<HealthCheckEventSinkSharedPtr> event_sinks_;
 };
 
 } // namespace Upstream

source/common/upstream/health_checker_impl.cc

@@ -47,44 +47,12 @@ const std::string& HealthCheckerFactory::getHostname(const HostSharedPtr& host,
   return cluster->name();
 }
 
-class HealthCheckerFactoryContextImpl : public Server::Configuration::HealthCheckerFactoryContext {
-public:
-  HealthCheckerFactoryContextImpl(Upstream::Cluster& cluster, Envoy::Runtime::Loader& runtime,
-                                  Event::Dispatcher& dispatcher,
-                                  HealthCheckEventLoggerPtr&& event_logger,
-                                  ProtobufMessage::ValidationVisitor& validation_visitor,
-                                  Api::Api& api)
-      : cluster_(cluster), runtime_(runtime), dispatcher_(dispatcher),
-        event_logger_(std::move(event_logger)), validation_visitor_(validation_visitor), api_(api) {
-  }
-  Upstream::Cluster& cluster() override { return cluster_; }
-  Envoy::Runtime::Loader& runtime() override { return runtime_; }
-  Event::Dispatcher& mainThreadDispatcher() override { return dispatcher_; }
-  HealthCheckEventLoggerPtr eventLogger() override { return std::move(event_logger_); }
-  ProtobufMessage::ValidationVisitor& messageValidationVisitor() override {
-    return validation_visitor_;
-  }
-  Api::Api& api() override { return api_; }
-
-private:
-  Upstream::Cluster& cluster_;
-  Envoy::Runtime::Loader& runtime_;
-  Event::Dispatcher& dispatcher_;
-  HealthCheckEventLoggerPtr event_logger_;
-  ProtobufMessage::ValidationVisitor& validation_visitor_;
-  Api::Api& api_;
-};
-
 HealthCheckerSharedPtr HealthCheckerFactory::create(
     const envoy::config::core::v3::HealthCheck& health_check_config, Upstream::Cluster& cluster,
     Runtime::Loader& runtime, Event::Dispatcher& dispatcher,
     AccessLog::AccessLogManager& log_manager,
     ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api) {
   HealthCheckEventLoggerPtr event_logger;
-  if (!health_check_config.event_log_path().empty()) {
-    event_logger = std::make_unique<HealthCheckEventLoggerImpl>(
-        log_manager, dispatcher.timeSource(), health_check_config.event_log_path());
-  }
   Server::Configuration::CustomHealthCheckerFactory* factory = nullptr;
 
   switch (health_check_config.health_checker_case()) {
@@ -112,9 +80,17 @@ HealthCheckerSharedPtr HealthCheckerFactory::create(
             health_check_config.custom_health_check());
   }
   }
+
   std::unique_ptr<Server::Configuration::HealthCheckerFactoryContext> context(
-      new HealthCheckerFactoryContextImpl(cluster, runtime, dispatcher, std::move(event_logger),
-                                          validation_visitor, api));
+      new HealthCheckerFactoryContextImpl(cluster, runtime, dispatcher, validation_visitor, api,
+                                          log_manager));
+
+  if (!health_check_config.event_log_path().empty() /* deprecated */ ||
+      !health_check_config.event_logger().empty()) {
+    event_logger = std::make_unique<HealthCheckEventLoggerImpl>(
+        log_manager, dispatcher.timeSource(), health_check_config, *context);
+  }
+  context->setEventLogger(std::move(event_logger));
   return factory->createCustomHealthChecker(health_check_config, *context);
 }