feat(opentelemetry sink): new sink (#21866)

* feat(opentelemetry sink): new sink

* docs: CUE, md, etc

* changelog

* allow word

* allow word

* Update website/cue/reference/components/sinks/base/opentelemetry.cue

Co-authored-by: Esther Kim <[email protected]>

* Update website/cue/reference/components/sinks/base/opentelemetry.cue

Co-authored-by: Esther Kim <[email protected]>

* Update website/cue/reference/components/sinks/base/opentelemetry.cue

Co-authored-by: Esther Kim <[email protected]>

* gen docs

* :cue: fmt

---------

Co-authored-by: Esther Kim <[email protected]>

.github/actions/spelling/allow.txt

@@ -472,3 +472,5 @@ zeek
 zookeeper
 zst
 zstandard
+otel
+otelcol

Cargo.toml

@@ -690,8 +690,8 @@ sinks-logs = [
   "sinks-aws_kinesis_firehose",
   "sinks-aws_kinesis_streams",
   "sinks-aws_s3",
-  "sinks-aws_sqs",
   "sinks-aws_sns",
+  "sinks-aws_sqs",
   "sinks-axiom",
   "sinks-azure_blob",
   "sinks-azure_monitor_logs",
@@ -712,12 +712,13 @@ sinks-logs = [
   "sinks-humio",
   "sinks-influxdb",
   "sinks-kafka",
-  "sinks-mezmo",
   "sinks-loki",
+  "sinks-mezmo",
   "sinks-mqtt",
   "sinks-nats",
-  "sinks-new_relic_logs",
   "sinks-new_relic",
+  "sinks-new_relic_logs",
+  "sinks-opentelemetry",
   "sinks-papertrail",
   "sinks-pulsar",
   "sinks-redis",
@@ -783,6 +784,7 @@ sinks-mqtt = ["dep:rumqttc"]
 sinks-nats = ["dep:async-nats", "dep:nkeys"]
 sinks-new_relic_logs = ["sinks-http"]
 sinks-new_relic = []
+sinks-opentelemetry = []
 sinks-papertrail = ["dep:syslog"]
 sinks-prometheus = ["dep:base64", "dep:prost", "vector-lib/prometheus"]
 sinks-pulsar = ["dep:apache-avro", "dep:pulsar", "dep:lru"]

changelog.d/21866_otel_sink.feature.md

@@ -0,0 +1,3 @@
+Introducing the first version of the [OpenTelemetry](https://opentelemetry.io/docs/what-is-opentelemetry/) sink.
+
+authors: pront

lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/README.md

@@ -1,2 +1,7 @@
 The following protobuf definitions are vendored from:
-https://github.com/open-telemetry/opentelemetry-proto/tree/v1.0.0/opentelemetry/proto
+
+* https://github.com/open-telemetry/opentelemetry-proto/tree/v1.0.0/opentelemetry/proto
+
+At the moment, these are manually updated based on community demand.
+Please [open an issue](https://github.com/vectordotdev/vector/issues/new?assignees=&labels=type%3A+feature&projects=&template=feature.yml)
+if you would like to see a newer version.

src/lib.rs

@@ -21,13 +21,15 @@
 
 //! The main library to support building Vector.
 
-#[macro_use]
-extern crate tracing;
 #[macro_use]
 extern crate derivative;
 #[macro_use]
+extern crate tracing;
+#[macro_use]
 extern crate vector_lib;
 
+pub use indoc::indoc;
+
 #[cfg(all(feature = "tikv-jemallocator", not(feature = "allocation-tracing")))]
 #[global_allocator]
 static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;

src/sinks/mod.rs

@@ -82,6 +82,8 @@ pub mod nats;
 pub mod new_relic;
 #[cfg(feature = "sinks-webhdfs")]
 pub mod opendal_common;
+#[cfg(feature = "sinks-opentelemetry")]
+pub mod opentelemetry;
 #[cfg(feature = "sinks-papertrail")]
 pub mod papertrail;
 #[cfg(feature = "sinks-prometheus")]

src/sinks/opentelemetry/mod.rs

@@ -0,0 +1,93 @@
+use crate::codecs::{EncodingConfigWithFraming, Transformer};
+use crate::config::{AcknowledgementsConfig, Input, SinkConfig, SinkContext};
+use crate::sinks::http::config::{HttpMethod, HttpSinkConfig};
+use crate::sinks::{Healthcheck, VectorSink};
+use indoc::indoc;
+use vector_config::component::GenerateConfig;
+use vector_lib::codecs::encoding::{FramingConfig, SerializerConfig};
+use vector_lib::codecs::JsonSerializerConfig;
+use vector_lib::configurable::configurable_component;
+
+/// Configuration for the `OpenTelemetry` sink.
+#[configurable_component(sink("opentelemetry", "Deliver OTLP data over HTTP."))]
+#[derive(Clone, Debug, Default)]
+pub struct OpenTelemetryConfig {
+    /// Protocol configuration
+    #[configurable(derived)]
+    protocol: Protocol,
+}
+
+/// The protocol used to send data to OpenTelemetry.
+/// Currently only HTTP is supported, but we plan to support gRPC.
+/// The proto definitions are defined [here](https://github.com/vectordotdev/vector/blob/master/lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry/proto/README.md).
+#[configurable_component]
+#[derive(Clone, Debug)]
+#[serde(rename_all = "snake_case", tag = "type")]
+#[configurable(metadata(docs::enum_tag_description = "The communication protocol."))]
+pub enum Protocol {
+    /// Send data over HTTP.
+    Http(HttpSinkConfig),
+}
+
+impl Default for Protocol {
+    fn default() -> Self {
+        Protocol::Http(HttpSinkConfig {
+            encoding: EncodingConfigWithFraming::new(
+                Some(FramingConfig::NewlineDelimited),
+                SerializerConfig::Json(JsonSerializerConfig::default()),
+                Transformer::default(),
+            ),
+            uri: Default::default(),
+            method: HttpMethod::Post,
+            auth: Default::default(),
+            headers: Default::default(),
+            compression: Default::default(),
+            payload_prefix: Default::default(),
+            payload_suffix: Default::default(),
+            batch: Default::default(),
+            request: Default::default(),
+            tls: Default::default(),
+            acknowledgements: Default::default(),
+        })
+    }
+}
+
+impl GenerateConfig for OpenTelemetryConfig {
+    fn generate_config() -> toml::Value {
+        toml::from_str(indoc! {r#"
+            [protocol]
+            type = "http"
+            uri = "http://localhost:5318/v1/logs"
+            encoding.codec = "json"
+        "#})
+        .unwrap()
+    }
+}
+
+#[async_trait::async_trait]
+#[typetag::serde(name = "opentelemetry")]
+impl SinkConfig for OpenTelemetryConfig {
+    async fn build(&self, cx: SinkContext) -> crate::Result<(VectorSink, Healthcheck)> {
+        match &self.protocol {
+            Protocol::Http(config) => config.build(cx).await,
+        }
+    }
+
+    fn input(&self) -> Input {
+        Input::all()
+    }
+
+    fn acknowledgements(&self) -> &AcknowledgementsConfig {
+        match self.protocol {
+            Protocol::Http(ref config) => config.acknowledgements(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    #[test]
+    fn generate_config() {
+        crate::test_util::test_generate_config::<super::OpenTelemetryConfig>();
+    }
+}

website/content/en/docs/reference/configuration/sinks/opentelemetry.md

@@ -0,0 +1,14 @@
+---
+title: OpenTelemetry
+description: Send [OTLP](https://opentelemetry.io/docs/reference/specification/protocol/otlp/) data through HTTP.
+component_kind: sink
+layout: component
+tags: [ "opentelemetry", "otel", "component", "sink", "logs", "metrics", "traces" ]
+---
+
+{{/*
+This doc is generated using:
+
+1. The template in layouts/docs/component.html
+2. The relevant CUE data in cue/reference/components/...
+   */}}